From b094a15ce1c6da7f7a91c92a5b5bb92583a25031 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:13:38 +0200 Subject: [PATCH 1/9] refactor: remove dead local-DC contact-point compatibility check (DRIVER-201) OptionalLocalDcHelper.checkLocalDatacenterCompatibility() warned when a contact point reported a datacenter different from the configured local DC. This has been dead code on scylla-4.x since 12e6acb90b: refresh matches nodes by hostId only, so contact-point nodes never get a datacenter assigned and the warning could never fire. Remove it. The separate "configured local DC matches no node" warning is retained. Nothing covered the removal, and CUSTOMER-588 is the bug the check caused: it compared the configured local DC against ephemeral placeholder Nodes (built by MetadataManager#addContactPoints via DefaultNode#newContactPoint, datacenter always null), so it warned unconditionally whenever a local DC was configured, no matter where the contact points actually were. The new test builds a placeholder Node the same way production does, plus a resolved node that genuinely is in the configured local DC, and asserts no warning is logged. It asserts on the absence of any WARN rather than of one particular message, so a regression under different wording is still caught; should_warn_if_configured_dc_matches_no_node is the positive control for the same appender, so a silent capture failure cannot make it pass by accident. Co-Authored-By: Claude Opus 5 (1M context) --- .../helper/OptionalLocalDcHelper.java | 74 +++++-------------- .../DefaultLoadBalancingPolicyInitTest.java | 36 +++++++++ 2 files changed, 53 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java index b93a16a6525..97aab92fff7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java @@ -26,7 +26,6 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -68,73 +67,34 @@ public OptionalLocalDcHelper( @Override @NonNull public Optional discoverLocalDc(@NonNull Map nodes) { - String localDcStr = context.getLocalDatacenter(profile.getName()); - Optional localDc; - if (localDcStr != null) { - LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); + String localDc = context.getLocalDatacenter(profile.getName()); + if (localDc != null) { + LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDc); } else if (profile.isDefined(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) { - localDcStr = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); - LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); - } else { - localDc = Optional.empty(); - } - if (localDc.isPresent()) { - checkLocalDatacenterCompatibility( - localDc.get(), context.getMetadataManager().getContactPoints()); - // Also warn if the configured DC doesn't match any node in the cluster - if (!nodes.isEmpty()) { - boolean found = false; - for (Node node : nodes.values()) { - if (localDc.get().equals(node.getDatacenter())) { - found = true; - break; - } - } - if (!found) { - LOG.warn( - "[{}] Configured local DC '{}' does not match any node's datacenter" - + " (available DCs: {}); please verify your configuration", - logPrefix, - localDc.get(), - formatDcs(nodes.values())); - } - } + localDc = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); + LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDc); } else { LOG.debug("[{}] Local DC not set, DC awareness will be disabled", logPrefix); + return Optional.empty(); } - return localDc; - } - - /** - * Checks if the contact points are compatible with the local datacenter specified either through - * configuration, or programmatically. - * - *

The default implementation logs a warning when a contact point reports a datacenter - * different from the local one, and only for the default profile. - * - * @param localDc The local datacenter, as specified in the config, or programmatically. - * @param contactPoints The contact points provided when creating the session. - */ - protected void checkLocalDatacenterCompatibility( - @NonNull String localDc, Set contactPoints) { - if (profile.getName().equals(DriverExecutionProfile.DEFAULT_NAME)) { - Set badContactPoints = new LinkedHashSet<>(); - for (Node node : contactPoints) { - if (!Objects.equals(localDc, node.getDatacenter())) { - badContactPoints.add(node); + if (!nodes.isEmpty()) { + boolean found = false; + for (Node node : nodes.values()) { + if (localDc.equals(node.getDatacenter())) { + found = true; + break; } } - if (!badContactPoints.isEmpty()) { + if (!found) { LOG.warn( - "[{}] You specified {} as the local DC, but some contact points are from a different DC: {}; " - + "please provide the correct local DC, or check your contact points", + "[{}] Configured local DC '{}' does not match any node's datacenter" + + " (available DCs: {}); please verify your configuration", logPrefix, localDc, - formatNodesAndDcs(badContactPoints)); + formatDcs(nodes.values())); } } + return Optional.of(localDc); } /** diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java index b5e843d77d2..9c36cbebfee 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -29,9 +30,12 @@ import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; import com.datastax.oss.driver.api.core.metadata.NodeState; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.InetSocketAddress; import java.util.UUID; import org.junit.Test; import org.junit.runner.RunWith; @@ -210,6 +214,38 @@ public void should_warn_if_configured_dc_matches_no_node() { .isTrue(); } + @Test + public void should_not_warn_about_dc_mismatch_when_the_only_real_node_matches_configured_dc() { + // Given — CUSTOMER-588. A contact point given as a hostname is represented, before the control + // connection resolves it, by an ephemeral placeholder Node (built by + // MetadataManager#addContactPoints via DefaultNode#newContactPoint) whose datacenter is always + // null: it is never populated, because real topology is attached to a *different* Node object + // matched by hostId (see MetadataManager#registerNode). + // + // The removed OptionalLocalDcHelper#checkLocalDatacenterCompatibility compared the configured + // local DC against *those* placeholders, so it warned unconditionally whenever a local DC was + // configured, no matter where the contact points actually were. Here the only node carrying + // real, resolved metadata (node1) genuinely is in the configured local DC ("dc1", per base + // setup). + DefaultNode ephemeralContactPointNode = + DefaultNode.newContactPoint( + new DefaultEndPoint(new InetSocketAddress("127.0.0.9", 9042)), context); + when(metadataManager.getContactPoints()).thenReturn(ImmutableSet.of(ephemeralContactPointNode)); + DefaultLoadBalancingPolicy policy = createPolicy(); + + // When + policy.init(ImmutableMap.of(UUID.randomUUID(), node1), distanceReporter); + + // Then — no WARN at all. The retained check inspects the resolved node map, where node1 + // matches. + // Asserting that nothing is warned, rather than that one particular message is absent, also + // catches a regression that brings the false positive back under different wording. + // should_warn_if_configured_dc_matches_no_node is the positive control for this same appender, + // so a silent capture failure cannot make this pass by accident. + verify(appender, never()).doAppend(argThat(event -> event.getLevel() == Level.WARN)); + assertThat(policy.getLocalDatacenter()).isEqualTo("dc1"); + } + @NonNull protected DefaultLoadBalancingPolicy createPolicy() { return new DefaultLoadBalancingPolicy(context, DriverExecutionProfile.DEFAULT_NAME); From 6e9ed8a3150d5c55cc73febba1c1b999c88933bd Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:15:06 +0200 Subject: [PATCH 2/9] feat: keep contact points unresolved; deprecate RESOLVE_CONTACT_POINTS (DRIVER-201) Contact points backed by a hostname are now always kept unresolved, so the connection layer can expand them to all their DNS-mapped IPs at connection time. SessionBuilder no longer reads RESOLVE_CONTACT_POINTS when merging contact points; the option is deprecated and has no effect. An already-resolved InetSocketAddress passed programmatically is still used as provided, with no further expansion. OptionsMap.fillWithDriverDefaults() still carries the option's reference.conf value so the defaults map stays complete, and is annotated accordingly -- the build treats deprecation warnings as errors. Co-Authored-By: Claude Opus 5 (1M context) --- .../api/core/config/DefaultDriverOption.java | 7 +++ .../driver/api/core/config/OptionsMap.java | 3 ++ .../api/core/config/TypedDriverOption.java | 11 +++- .../api/core/session/SessionBuilder.java | 50 +++++++++++++++---- core/src/main/resources/reference.conf | 32 ++++++------ 5 files changed, 76 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index dd60a2487fb..3be41810660 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -837,7 +837,14 @@ public enum DefaultDriverOption implements DriverOption { * Whether to resolve the addresses passed to `basic.contact-points`. * *

Value-type: boolean + * + * @deprecated Setting this option has no effect. Contact points given in the configuration are + * now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily + * at connection time. This never applied to programmatic contact points passed to {@code + * SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved + * address stays bound to that one IP. */ + @Deprecated RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"), /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index c1a428b3524..293231347fe 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -245,6 +245,9 @@ private void readObject(ObjectInputStream stream) throws InvalidObjectException throw new InvalidObjectException("Proxy required"); } + // RESOLVE_CONTACT_POINTS is deprecated and has no effect, but it is still a driver option, so the + // defaults map stays complete by carrying its reference.conf value. + @SuppressWarnings("deprecation") protected static void fillWithDriverDefaults(OptionsMap map) { Duration initQueryTimeout = Duration.ofSeconds(5); Duration requestTimeout = Duration.ofSeconds(2); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index af93e734ef1..b0a5e788c67 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -664,7 +664,16 @@ public String toString() { /** The coalescer reschedule interval. */ public static final TypedDriverOption COALESCER_INTERVAL = new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION); - /** Whether to resolve the addresses passed to `basic.contact-points`. */ + /** + * Whether to resolve the addresses passed to `basic.contact-points`. + * + * @deprecated Setting this option has no effect. Contact points given in the configuration are + * now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily + * at connection time. This never applied to programmatic contact points passed to {@code + * SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved + * address stays bound to that one IP. + */ + @Deprecated public static final TypedDriverOption RESOLVE_CONTACT_POINTS = new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN); /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index 8375f0ef30b..db23692ea16 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -166,11 +166,15 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad *

Contact points can also be provided statically in the configuration. If both are specified, * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042. * - *

Contrary to the configuration, DNS names with multiple A-records will not be handled here. - * If you need that, extract them manually with {@link java.net.InetAddress#getAllByName(String)} - * before calling this method. Similarly, if you need connect addresses to stay unresolved, make - * sure you pass unresolved instances here (see {@code advanced.resolve-contact-points} in the - * configuration for more explanations). + *

The driver automatically expands any contact point backed by an unresolved hostname to all + * its DNS-mapped IPs at connection time (through Netty's configured resolver, so a custom {@code + * AddressResolverGroup} still applies), so passing a single hostname is sufficient to try all its + * IPs on initial connect. This applies equally to hostnames provided here programmatically (build + * an unresolved {@link InetSocketAddress} with {@link InetSocketAddress#createUnresolved(String, + * int)} to opt in) and to hostnames specified in the configuration. An already-resolved address + * passed here (the common case when constructing an {@code InetSocketAddress} directly from a + * hostname, which resolves eagerly) is used as provided, with no further expansion. The {@code + * advanced.resolve-contact-points} option is deprecated and has no effect. */ @NonNull public SelfT addContactPoints(@NonNull Collection contactPoints) { @@ -741,6 +745,12 @@ public SelfT withCloudSecureConnectBundle(@NonNull InputStream cloudConfigInputS * *

For more information, please refer to the DataStax Astra documentation. * + *

A proxy given as a hostname is resolved at connection time, to all of its addresses, + * and each is tried in turn. That holds however the {@link InetSocketAddress} was built: the + * driver keeps a proxy hostname unresolved internally, so passing one that the ordinary {@code + * InetSocketAddress(String, int)} constructor already resolved does not bind the session to that + * single address. + * * @param cloudProxyAddress The address of the Cloud proxy to use. * @see Server Name Indication */ @@ -957,11 +967,15 @@ protected final CompletionStage buildDefaultSessionAsync() { programmaticArguments = programmaticArgumentsBuilder.build(); } - boolean resolveAddresses = - defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false); - + // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved + // hostnames, and expanded to all their DNS IPs at connection time by ChannelFactory. + // The value is still read, only to tell someone who set it that it no longer does anything. + // Note this tests the value rather than isDefined(): unlike the deprecated options warned + // about in DefaultDriverContext, this one ships uncommented in reference.conf, so it is + // always defined. + warnIfResolveContactPointsRequested(defaultConfig); Set contactPoints = - ContactPoints.merge(programmaticContactPoints, configContactPoints, resolveAddresses); + ContactPoints.merge(programmaticContactPoints, configContactPoints, false); if (keyspace == null && defaultConfig.isDefined(DefaultDriverOption.SESSION_KEYSPACE)) { keyspace = @@ -989,6 +1003,24 @@ private boolean anyProfileHasDatacenterDefined(DriverConfig driverConfig) { return false; } + /** + * Tells anyone who turned {@code advanced.resolve-contact-points} on that it no longer does + * anything, so the behaviour they configured does not disappear in silence. + */ + @SuppressWarnings("deprecation") + private static void warnIfResolveContactPointsRequested(DriverExecutionProfile defaultConfig) { + if (defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false)) { + LOG.warn( + "Option {} is deprecated and no longer has any effect. Contact points given in the" + + " configuration are now always kept as unresolved hostnames and expanded to all of" + + " their addresses at connection time, so a name that resolves to several nodes is" + + " one Node in the driver's metadata rather than one per address. Note that this" + + " never applied to contact points passed to addContactPoints(): those are used" + + " exactly as supplied, and an already-resolved address stays bound to that one IP.", + DefaultDriverOption.RESOLVE_CONTACT_POINTS.getPath()); + } + } + /** * Returns URL based on the configUrl setting. If the configUrl has no protocol provided, the * method will fallback to file:// protocol and return URL that has file protocol specified. diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 590784b70c5..4a13886c9d9 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1233,27 +1233,25 @@ datastax-java-driver { } - # Whether to resolve the addresses passed to `basic.contact-points`. + # DEPRECATED: this option no longer has any effect and will be removed in a future release. # - # If this is true, addresses are created with `InetSocketAddress(String, int)`: the host name will - # be resolved the first time, and the driver will use the resolved IP address for all subsequent - # connection attempts. + # Contact points given here are now always kept as unresolved hostnames and expanded to all of + # their DNS-mapped IPs lazily at connection time. This means the driver tries every IP a hostname + # resolves to, and re-resolves the hostname on each new connection so DNS changes are picked up + # automatically. Previously this option selected between resolving a contact-point hostname once + # (true) and re-resolving it on every connection (false); that distinction no longer applies. # - # If this is false, addresses are created with `InetSocketAddress.createUnresolved()`: the host - # name will be resolved again every time the driver opens a new connection. This is useful for - # containerized environments where DNS records are more likely to change over time (note that the - # JVM and OS have their own DNS caching mechanisms, so you might need additional configuration - # beyond the driver). + # The lookup goes through Netty's configured AddressResolverGroup -- the same resolver an + # unresolved address would have reached had it been passed straight to Bootstrap.connect() -- so a + # custom resolver installed via NettyOptions.afterBootstrapInitialized() still applies. With + # Netty's default (JDK) resolver the lookup blocks the I/O event loop it runs on; install + # DnsAddressResolverGroup if you need it to be non-blocking. # - # This option only applies to the contact points specified in the configuration. It has no effect - # on: - # - programmatic contact points passed to SessionBuilder.addContactPoints: these addresses are - # built outside of the driver, so it is your responsibility to provide unresolved instances. - # - dynamically discovered peers: the driver relies on Cassandra system tables, which expose raw - # IP addresses. Use a custom address translator to convert them to unresolved addresses (if - # you're in a containerized environment, you probably already need address translation anyway). + # This option only ever applied to the contact points specified in the configuration -- never to + # programmatic contact points passed to SessionBuilder.addContactPoints, nor to dynamically + # discovered peers. # - # Required: no (defaults to false) + # Required: no # Modifiable at runtime: no # Overridable in a profile: no advanced.resolve-contact-points = false From 5c28d94d4bf5ec9861ab2ff7f8f37d992fdd16ed Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:19:54 +0200 Subject: [PATCH 3/9] feat: let the endpoint layer hand out unresolved addresses (DRIVER-201) Groundwork for expanding a hostname to all of its addresses: resolution becomes the connection layer's job, so everything that produces an EndPoint stops doing DNS of its own, and an endpoint gains a way to record which address a connection actually reached. PinnableEndPoint is the new internal contract: pinTo(SocketAddress) returns a copy bound to one address, and the pin is excluded from equals(), hashCode(), asMetricPrefix() and toString(). Endpoints are set and map keys, and node metrics are named after them, so a pinned copy has to be indistinguishable from its original everywhere except when the connection layer asks which address answered. A generic delegating wrapper was rejected: its equals() would be asymmetric, because DefaultEndPoint#equals tests instanceof and would reject the wrapper while the wrapper accepted the original, and it would break SniSslEngineFactory's instanceof SniEndPoint guard. Each implementation therefore carries a nullable pinnedAddress of its own. SniEndPoint additionally normalizes a resolved proxy *hostname* back to unresolved. withCloudProxyAddress(new InetSocketAddress("proxy", 9042)) resolves eagerly, which froze Cloud on whichever proxy IP the JVM happened to return; an IP-literal proxy is left alone. Contact points keep the opposite policy on purpose, since ContactPoints.merge() only ever applied its resolve flag to config-file entries. ClientRoutesTopologyMonitor.resolve() likewise returns the client route as an unresolved address and no longer looks it up, which keeps it a pure in-memory cache lookup that is safe to call from an event loop, and lets a custom resolver apply to client routes just as it does to contact points. Its protected resolveAddress() extension point, which existed only so tests could stub out InetAddress.getByName, goes with it. This has to move together with ClientRoutesEndPoint: dropping "throws UnknownHostException" from one and the matching catch from the other is a single compilable change. TopologyMonitor gains reresolvesNodeAddresses(), which tells the control connection's reconnection query plan whether this monitor already keeps addresses fresh. It defaults to false, correct for DefaultTopologyMonitor, whose peers hold an already-resolved IP from system.peers. ClientRoutesTopologyMonitor reports true only when every currently-known node actually has a live route: where the route set is incomplete, ClientRoutesEndPoint falls back to a static resolved endpoint, and those nodes still need the contact-point fallback. The "is this a name" test several of these need is shared as AddressUtils.carriesName(): a resolved address compares its host string against the literal its own bytes produce, an unresolved one parses its host string. Neither isUnresolved() nor the presence of an InetAddress can tell a name from a literal on its own. DseGssApiAuthProviderBase.serverName() falls back to getHostString() when getAddress() returns null, which is now the ordinary case for a Cloud or client-route endpoint rather than an impossible one. EndPoint.resolve() keeps its signature and is not deprecated, so third-party implementations still compile. Its javadoc gains two expectations: return the address as-is rather than looking names up, since this is now called from an event loop; and callers are warned that the returned address is no longer always resolved, so getHostString() is the safe way to read the host. Co-Authored-By: Claude Opus 5 (1M context) --- .../core/auth/DseGssApiAuthProviderBase.java | 20 +- .../driver/api/core/metadata/EndPoint.java | 41 +++- .../api/core/session/SessionBuilder.java | 11 ++ .../core/metadata/ClientRoutesEndPoint.java | 95 ++++++++- .../metadata/ClientRoutesTopologyMonitor.java | 43 ++-- .../core/metadata/CloudTopologyMonitor.java | 10 + .../core/metadata/DefaultEndPoint.java | 110 ++++++++++- .../core/metadata/PinnableEndPoint.java | 153 +++++++++++++++ .../internal/core/metadata/SniEndPoint.java | 155 +++++++++++---- .../core/metadata/TopologyMonitor.java | 38 ++++ .../internal/core/util/AddressUtils.java | 33 ++++ .../metadata/ClientRoutesEndPointTest.java | 82 +++++++- .../ClientRoutesTopologyMonitorTest.java | 68 ++++++- .../core/metadata/DefaultEndPointTest.java | 138 +++++++++++++ .../core/metadata/SniEndPointTest.java | 183 ++++++++++++++++++ .../internal/core/util/AddressUtilsTest.java | 60 ++++++ 16 files changed, 1153 insertions(+), 87 deletions(-) create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java diff --git a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java index 48a0e5b0ef3..beab2f488a8 100644 --- a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java +++ b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java @@ -27,6 +27,7 @@ import com.datastax.oss.protocol.internal.util.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedActionException; @@ -319,7 +320,7 @@ protected GssApiAuthenticator( SUPPORTED_MECHANISMS, options.getAuthorizationId(), protocol, - ((InetSocketAddress) endPoint.resolve()).getAddress().getCanonicalHostName(), + serverName(endPoint), options.getSaslProperties(), null); } catch (LoginException | SaslException e) { @@ -328,6 +329,23 @@ protected GssApiAuthenticator( this.endPoint = endPoint; } + /** + * The host name to build the Kerberos service principal from. + * + *

Prefers the canonical name of the resolved address, which is what Kerberos expects. The + * driver's own endpoints always hand this a resolved address — the channel carries an endpoint + * bound to the address it connected to (see {@code PinnableEndPoint}) — but a custom {@link + * EndPoint} implementation may still yield an unresolved one, in which case {@code + * getAddress()} is null. Fall back to the host string rather than throwing a {@link + * NullPointerException}: the hostname is usually the right service name anyway, and a failed + * reverse lookup should not take authentication down. + */ + private static String serverName(EndPoint endPoint) { + InetSocketAddress address = (InetSocketAddress) endPoint.resolve(); + InetAddress inetAddress = address.getAddress(); + return inetAddress != null ? inetAddress.getCanonicalHostName() : address.getHostString(); + } + @NonNull @Override protected ByteBuffer getMechanism() { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java index 530f2ad38ac..06dea1daef2 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java @@ -18,24 +18,59 @@ package com.datastax.oss.driver.api.core.metadata; import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.InetSocketAddress; import java.net.SocketAddress; /** * Encapsulates the information needed to open connections to a node. * *

By default, the driver assumes plain TCP connections, and this is just a wrapper around an - * {@link InetSocketAddress}. However, more complex deployment scenarios might use a custom + * {@link java.net.InetSocketAddress}. However, more complex deployment scenarios might use a custom * implementation that contains additional information; for example, if the nodes are accessed * through a proxy with SNI routing, an SNI server name is needed in addition to the proxy address. */ public interface EndPoint { /** - * Resolves this instance to a socket address. + * Resolves this instance to the socket address connections should be opened to. * *

This will be called each time the driver opens a new connection to the node. The returned * address cannot be null. + * + *

Returning a hostname is fine, and is how multi-address support works. The returned + * address need not be resolved: an {@linkplain java.net.InetSocketAddress#isUnresolved() + * unresolved} {@link java.net.InetSocketAddress} is expanded by the driver to every + * address the name maps to, and each one is tried in turn until a connection succeeds. That is + * what {@link com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint} does for contact + * points backed by a hostname, so a single unreachable IP behind a multi-record name no longer + * fails the connection. + * + *

Implementations must not resolve names themselves, and must not block. The driver + * calls this from its admin event loop, and it performs the expansion through Netty's configured + * {@code AddressResolverGroup} — the same resolver an unresolved address reaches when it is + * handed to {@code Bootstrap.connect()}. Looking the name up here instead (for example with + * {@link java.net.InetAddress#getAllByName(String)}) would both block that loop and bypass a + * custom resolver installed via {@code NettyOptions#afterBootstrapInitialized(Bootstrap)}. + * + *

Callers must not assume the returned address is resolved. It is for a node discovered + * from {@code system.peers} (built from that node's physical broadcast RPC address) and for the + * node the control connection is on (bound to the address that connection reached). It is + * not for a node reached through the Cloud SNI proxy, or through a cloud private-endpoint + * client route: there the address is the configured hostname, and {@link + * java.net.InetSocketAddress#getAddress()} returns {@code null}. Read the host with {@link + * java.net.InetSocketAddress#getHostString()}, which yields whichever of the two the address + * carries and never triggers a reverse lookup. + * + * @apiNote Timeout note: when a name expands to several addresses they are tried in + * sequence, so the worst-case time before the node is declared unreachable is N times a full + * attempt — and an attempt is more than a connect. Each address that accepts the TCP + * connection then runs the init handshake, whose steps each arm their own {@code + * advanced.connection.init-query-timeout}; those add up rather than sharing one deadline. An + * address that stalls after accepting the connection can therefore burn {@code + * advanced.connection.connect-timeout} plus several times {@code + * advanced.connection.init-query-timeout} on its own. In practice DNS round-robin entries + * have only a small number of records, so this is rarely a concern, but it is worth bearing + * in mind when configuring timeouts — note also that session initialization has no overall + * deadline of its own. */ @NonNull SocketAddress resolve(); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index db23692ea16..409ac5a589f 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -751,6 +751,17 @@ public SelfT withCloudSecureConnectBundle(@NonNull InputStream cloudConfigInputS * InetSocketAddress(String, int)} constructor already resolved does not bind the session to that * single address. * + *

Prefer {@link InetSocketAddress#createUnresolved(String, int)} all the same, and especially + * for a proxy given as an IP address. Whether an address carries a name is read from + * {@code getHostString()}, which falls back to the underlying {@link java.net.InetAddress}'s + * cached host name -- and that field is filled in, on the very instance passed here, the first + * time anything calls {@code getHostName()} on it. The SNI SSL engine does exactly that while + * building an engine, unless reverse-lookup SANs are turned off. So an IP that has a {@code PTR} + * record can acquire a name mid-session, after which the driver treats that name as the proxy: + * the endpoints it builds from then on compare unequal to the earlier ones, report metrics under + * a different prefix, and connect to wherever that name resolves. An unresolved address is never + * subject to this, and is what the secure connect bundle produces. + * * @param cloudProxyAddress The address of the Cloud proxy to use. * @see Server Name Indication */ diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java index 15d825b2efc..134b813bc48 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -20,19 +20,26 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; import java.util.UUID; -public class ClientRoutesEndPoint implements EndPoint { +public class ClientRoutesEndPoint implements PinnableEndPoint { private final UUID hostId; private final ClientRoutesTopologyMonitor topologyMonitor; private final String metricPrefix; @NonNull private final EndPoint fallbackEndPoint; + /** Kept only so that {@link #pinTo(SocketAddress)} can rebuild an identical copy. */ + @Nullable private final InetAddress broadcastInetAddress; + + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}, + * which key off the host id alone. + */ + @Nullable private final InetSocketAddress pinnedAddress; /** * @param topologyMonitor the topology monitor used to resolve the endpoint address on demand. @@ -49,12 +56,23 @@ public ClientRoutesEndPoint( @NonNull UUID hostId, @Nullable InetAddress broadcastInetAddress, @NonNull EndPoint fallbackEndPoint) { + this(topologyMonitor, hostId, broadcastInetAddress, fallbackEndPoint, null); + } + + private ClientRoutesEndPoint( + @NonNull ClientRoutesTopologyMonitor topologyMonitor, + @NonNull UUID hostId, + @Nullable InetAddress broadcastInetAddress, + @NonNull EndPoint fallbackEndPoint, + @Nullable InetSocketAddress pinnedAddress) { this.topologyMonitor = Objects.requireNonNull(topologyMonitor, "Topology monitor cannot be null"); this.hostId = Objects.requireNonNull(hostId, "HOST uuid cannot be null"); this.fallbackEndPoint = Objects.requireNonNull(fallbackEndPoint, "Fallback endpoint cannot be null"); this.metricPrefix = buildMetricPrefix(broadcastInetAddress, hostId); + this.broadcastInetAddress = broadcastInetAddress; + this.pinnedAddress = pinnedAddress; } @NonNull @@ -62,18 +80,75 @@ public UUID getHostId() { return hostId; } + /** + * Returns the address connections should be opened to. + * + *

The client route for this host id is an in-memory lookup over the cached {@code + * system.client_routes} contents, and it yields exactly one address by design, so this neither + * blocks nor expands to several candidates. The route's hostname is returned {@linkplain + * InetSocketAddress#isUnresolved() unresolved}: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup}, so a custom resolver is honoured and no DNS lookup + * runs on the caller (the admin event loop, for control-connection reconnects). + * + *

When the topology monitor has no route for this host id — i.e. the node is not reached + * through a cloud private endpoint — this delegates to the fallback endpoint. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} the pinned address is returned directly. + */ @NonNull @Override public SocketAddress resolve() { + if (pinnedAddress != null) { + return pinnedAddress; + } + InetSocketAddress address; try { - InetSocketAddress address = topologyMonitor.resolve(hostId); - if (address != null) { - return address; - } - } catch (IOException e) { - throw new UncheckedIOException("DNS resolution failed for host_id=" + hostId, e); + address = topologyMonitor.resolve(hostId); + } catch (IllegalStateException e) { + // The monitor is closed, so its route cache is gone -- but resolve() still has to answer, and + // the honest answer is "no route available", which is what the fallback endpoint is for. + // Throwing here is not contained anywhere useful: PinnableEndPoint#sameIdentity compares + // resolve() results for every node of every topology refresh, and neither NodesRefresh nor + // MetadataManager#apply catches, so a refresh that raced session shutdown would be dropped + // whole and surface only as a DEBUG log in ControlConnection#onSuccessfulReconnect. + address = null; } - return fallbackEndPoint.resolve(); + return address != null ? address : fallbackEndPoint.resolve(); + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + // Mirror DefaultEndPoint: an address we cannot hold in an InetSocketAddress field skips + // pinning rather than failing the connection. So does an unresolved one -- resolve() hands out + // the route's hostname unresolved, and ChannelFactory passes it straight back when the user + // disabled the resolver or a custom one declines it. Pinning that would freeze the endpoint on + // a name that still re-expands on every connect: no address stability gained, and the route + // lookup silenced for good, since resolve() short-circuits once pinned. + if (!(resolvedAddress instanceof InetSocketAddress) + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress)) { + return this; + } + return new ClientRoutesEndPoint( + topologyMonitor, + hostId, + broadcastInetAddress, + fallbackEndPoint, + (InetSocketAddress) resolvedAddress); + } + + /** + * {@inheritDoc} + * + *

{@code true}: a route's addresses are alternative ways in to this one node, so connections + * may be spread across them. + */ + @Override + public boolean addressesAreInterchangeable() { + return true; } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java index 1ffc35fd9f4..31d27aa7c7d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java @@ -21,6 +21,7 @@ import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; @@ -32,7 +33,6 @@ import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.UnknownHostException; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -195,9 +195,18 @@ void setResolvedRoutes(Map routes) { resolvedRoutesCache.set(Collections.unmodifiableMap(new HashMap<>(routes))); } + /** + * Returns the client route for {@code hostId} as an {@linkplain InetSocketAddress#isUnresolved() + * unresolved} address, or {@code null} if this node has no route. + * + *

The route's hostname is deliberately left unresolved: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup} at connection time. That keeps this method a pure + * in-memory cache lookup, so it is safe to call from an event loop, and it means a custom + * resolver applies to client routes just like it does to contact points. + */ @Nullable - public InetSocketAddress resolve(@NonNull UUID hostId) - throws IllegalStateException, UnknownHostException { + public InetSocketAddress resolve(@NonNull UUID hostId) throws IllegalStateException { if (closed) { throw new IllegalStateException("Topology monitor is closed"); } @@ -206,7 +215,7 @@ public InetSocketAddress resolve(@NonNull UUID hostId) return null; // no client route for this node — caller falls back to default } - return new InetSocketAddress(resolveAddress(route.getHostname()), route.getPort()); + return InetSocketAddress.createUnresolved(route.getHostname(), route.getPort()); } /** @@ -480,6 +489,23 @@ protected EndPoint buildNodeEndPoint( return new ClientRoutesEndPoint(this, hostId, broadcastInetAddress, fallback); } + @Override + public boolean reresolvesNodeAddresses() { + // ClientRoutesEndPoint hands the route hostname over unresolved, so the connection layer + // re-expands it on every connection attempt -- but only when a route exists for that host_id + // (see ClientRoutesEndPoint#resolve()); for mixed/incomplete route sets it delegates to a + // static, already-resolved fallback endpoint instead. Only report true when every + // currently-known node actually has a live route; otherwise the contact-point reconnection + // fallback must stay available for the nodes stuck on that fallback. + Map routes = resolvedRoutesCache.get(); + for (Node node : context.getMetadataManager().getMetadata().getNodes().values()) { + if (!routes.containsKey(node.getHostId())) { + return false; + } + } + return true; + } + /** * Builds the CQL query to fetch client routes. * @@ -645,13 +671,4 @@ public CompletionStage closeAsync() { LOG.debug("[{}] ClientRoutesTopologyMonitor closed", logPrefix); return super.closeAsync(); } - - /** - * Resolves a hostname to an {@link InetAddress}. Extracted as a protected method so that unit - * tests can override it to return stubbed addresses without hitting the network. - */ - @NonNull - protected InetAddress resolveAddress(@NonNull String hostname) throws UnknownHostException { - return InetAddress.getByName(hostname); - } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java index 021824a9b16..7bdf1c4e1ec 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java @@ -44,4 +44,14 @@ protected EndPoint buildNodeEndPoint( UUID hostId = Objects.requireNonNull(row.getUuid("host_id")); return new SniEndPoint(cloudProxyAddress, hostId.toString()); } + + @Override + public boolean reresolvesNodeAddresses() { + // Every node is reached through the cloud SNI proxy, and SniEndPoint hands the proxy hostname + // over unresolved, so the connection layer re-expands it on every connection attempt (see + // ChannelFactory#resolveCandidates). Addresses therefore stay current on their own: appending + // the original contact points as a DNS re-resolution fallback would add nothing, and could + // resurrect nodes this monitor has authoritatively removed. + return true; + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index 7ffbee8e4bb..ffe2e3917b6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -19,28 +19,104 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.io.Serializable; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class DefaultEndPoint implements EndPoint, Serializable { +public class DefaultEndPoint implements PinnableEndPoint, Serializable { private static final long serialVersionUID = 1; + private static final Logger LOG = LoggerFactory.getLogger(DefaultEndPoint.class); + + /** Static, so the warning below is emitted once per JVM rather than once per endpoint. */ + private static final AtomicBoolean LOGGED_MIXED_COMPARISON_WARNING = new AtomicBoolean(); + private final InetSocketAddress address; private final String metricPrefix; + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals}, {@link #hashCode} and + * {@link #asMetricPrefix()}: a pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + public DefaultEndPoint(InetSocketAddress address) { + this(address, null); + } + + private DefaultEndPoint(InetSocketAddress address, @Nullable InetSocketAddress pinnedAddress) { this.address = Objects.requireNonNull(address, "address can't be null"); this.metricPrefix = buildMetricPrefix(address); + this.pinnedAddress = pinnedAddress; } + /** + * Returns the address connections should be opened to: the {@linkplain #pinTo(SocketAddress) + * pinned} one if this is a pinned copy, otherwise the stored address as-is. + * + *

This performs no name resolution. If the stored address is a hostname (i.e. {@linkplain + * InetSocketAddress#isUnresolved() unresolved} — contact points are always kept unresolved, see + * {@link com.datastax.oss.driver.api.core.session.SessionBuilder#addContactPoint}) it is returned + * unresolved, and {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands it + * to every IP it maps to through Netty's configured {@code AddressResolverGroup}. Resolving there + * rather than here is deliberate: it keeps any custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized} in the + * loop, which a direct {@code InetAddress.getAllByName()} call from here would bypass, and it + * keeps this method non-blocking so it is safe to call from an event loop. + */ @NonNull @Override public InetSocketAddress resolve() { - return address; + return pinnedAddress != null ? pinnedAddress : address; } + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress can't be null"); + if (!(resolvedAddress instanceof InetSocketAddress) + // An unresolved address, as ClientRoutesEndPoint and SniEndPoint also refuse: this endpoint + // hands a hostname over unresolved and ChannelFactory passes it straight back when the user + // disabled the resolver or a custom one declines it. Pinning that would freeze resolve() on + // a name that must re-expand on every connect. + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress) + // The address we already hold: pinning to it changes nothing, since resolve() and + // toString() would keep yielding what they already do. Skipping the copy spares toString() + // a + // redundant "addr(addr)" suffix on every already-resolved endpoint -- which is all of them, + // once a node is discovered from the peers rows. + || resolvedAddress.equals(this.address)) { + return this; + } + return new DefaultEndPoint(address, (InetSocketAddress) resolvedAddress); + } + + /** + * Whether {@code other} denotes the same node: the stored addresses are compared, ignoring which + * one either endpoint may be {@linkplain #pinTo(SocketAddress) pinned} to. + * + *

Comparing an unresolved address against a resolved one costs a DNS lookup, taken + * inline on the calling thread, because the unresolved side has to be resolved first. It is also + * arbitrary: {@code new InetSocketAddress(name, port)} keeps only the first address the + * name maps to, so for a multi-record name the answer is "equal iff this node is the one the + * resolver happened to list first". And it does not agree with {@link #hashCode()}, which keys on + * the stored address alone -- so a hostname and one of its IPs can be {@code equals} while + * hashing differently, and a hash-based collection of endpoints (the contact-point {@code Set}, + * for one) never treats them as the same entry. + * + *

No driver-internal path compares the two forms; the branch is kept for {@link + * com.datastax.oss.driver.api.core.metadata.Metadata#findNode(EndPoint)}, whose caller may hold + * either. It warns once per JVM, as a canary for whether anything still depends on it -- see + * https://github.com/scylladb/java-driver/issues/1006. + */ @Override public boolean equals(Object other) { if (other == this) { @@ -48,12 +124,15 @@ public boolean equals(Object other) { } else if (other instanceof DefaultEndPoint) { InetSocketAddress thisAddress = this.address; InetSocketAddress thatAddress = ((DefaultEndPoint) other).address; - // If only one of the addresses is unresolved, resolve the other. Otherwise (both resolved or - // both unresolved), compare as-is. - if (thisAddress.isUnresolved() && !thatAddress.isUnresolved()) { - thisAddress = new InetSocketAddress(thisAddress.getHostName(), thisAddress.getPort()); - } else if (thatAddress.isUnresolved() && !thisAddress.isUnresolved()) { - thatAddress = new InetSocketAddress(thatAddress.getHostName(), thatAddress.getPort()); + // If only one of the addresses is unresolved, resolve it. Otherwise (both resolved or both + // unresolved), compare as-is. + if (thisAddress.isUnresolved() != thatAddress.isUnresolved()) { + warnAboutMixedComparison(thisAddress, thatAddress); + if (thisAddress.isUnresolved()) { + thisAddress = new InetSocketAddress(thisAddress.getHostName(), thisAddress.getPort()); + } else { + thatAddress = new InetSocketAddress(thatAddress.getHostName(), thatAddress.getPort()); + } } return thisAddress.equals(thatAddress); } else { @@ -61,6 +140,18 @@ public boolean equals(Object other) { } } + private static void warnAboutMixedComparison(InetSocketAddress one, InetSocketAddress other) { + if (LOGGED_MIXED_COMPARISON_WARNING.compareAndSet(false, true)) { + LOG.warn( + "Compared an unresolved endpoint address against a resolved one ({} vs {}). This performs" + + " a DNS lookup on the calling thread, only compares the first address the name maps" + + " to, and does not agree with hashCode(); see" + + " https://github.com/scylladb/java-driver/issues/1006. This message is logged once.", + one, + other); + } + } + @Override public int hashCode() { return address.hashCode(); @@ -68,6 +159,9 @@ public int hashCode() { @Override public String toString() { + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which IP a given connection + // landed on is in the channel's own toString(), which Netty builds from the actual remote + // address. return address.toString(); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java new file mode 100644 index 00000000000..4154250ce4b --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.SocketAddress; +import java.util.Objects; + +/** + * An {@link EndPoint} that can produce a copy of itself bound ("pinned") to one specific address. + * + *

An endpoint whose hostname maps to several IPs describes a set of candidate addresses, + * but a channel is always connected to exactly one of them. {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} pins the endpoint to the address it + * actually used, and hands the pinned copy to the channel. That matters for two reasons: + * + *

    + *
  • Node identity. Once the driver has learnt, over a given connection, that {@code + * host_id} X answers at a given IP, that node must keep reconnecting to that IP. If + * the node kept the multi-address endpoint, a later reconnect could land on a different node + * while still being treated as X (see {@code DefaultTopologyMonitor#buildNodeEndPoint} and + * {@code ControlConnection}, which skip identity re-resolution for nodes that already have a + * host id). + *
  • No re-resolution on the channel path. Components handed the channel's endpoint call + * {@link EndPoint#resolve()} — SSL engine creation, GSSAPI service-name lookup, {@code + * DefaultTopologyMonitor#savePort}. On a pinned endpoint that is a field read, so it neither + * blocks on DNS (SSL setup runs on a Netty event loop) nor risks picking a different address + * than the one the channel is connected to. + *
+ * + *

This is an internal extension point: {@code ChannelFactory} pins endpoints that implement it + * and leaves any other implementation untouched, so third-party {@link EndPoint}s keep working + * exactly as before. + * + *

Implementations must keep {@link Object#equals}, {@link Object#hashCode}, {@link + * EndPoint#asMetricPrefix()} and {@link Object#toString()} identical to the unpinned + * original: a pinned copy denotes the same node, and every one of those is part of how the node is + * identified from the outside. Metric names in particular must not change depending on which IP a + * connection happened to land on — and that includes {@code toString()}, which is what {@code + * TaggingMetricIdGenerator} tags node metrics with, and what any third-party {@code + * MetricIdGenerator} is equally free to use. Nodes do adopt pinned copies (see {@code + * DefaultNode#setEndPoint}), so an identity that varied with the pin would silently re-tag a node's + * metrics mid-session. Equality must also stay symmetric: {@code original.equals(pinned)} and + * {@code pinned.equals(original)} must agree, since endpoints are used as set and map keys. + * + *

The pinned address is therefore observable only through {@link EndPoint#resolve()}. That is no + * loss for diagnostics: the address a channel is actually connected to appears in the channel's own + * {@code toString()}, which Netty builds from its remote address, and {@code ChannelFactory} logs + * each candidate as it tries it. + */ +public interface PinnableEndPoint extends EndPoint { + + /** + * Returns a copy of this endpoint that resolves to exactly {@code resolvedAddress}. + * + *

Implementations may return {@code this} when pinning does not apply (for example when the + * address is not of a type they can hold on to), or when it would be a no-op because the endpoint + * already resolves to exactly that address. + * + * @param resolvedAddress the address a connection was successfully established to; must not be + * null and must already be resolved. + */ + @NonNull + EndPoint pinTo(@NonNull SocketAddress resolvedAddress); + + /** + * Whether the addresses this endpoint expands to are interchangeable, i.e. reaching any one of + * them is reaching the same node. + * + *

This is what decides whether {@code ChannelFactory} may spread connections across them. The + * question is a property of what the name denotes, and it splits the name-based endpoints + * in two: + * + *

    + *
  • A front door — an SNI proxy, a cloud private-endpoint route — publishes several + * addresses that all lead to the same node by construction: the proxy routes by server + * name, not by which of its own IPs the client picked. Spreading across them is the whole + * point of publishing more than one, and it is what the driver did before multi-address + * support, when {@code SniEndPoint#resolve()} rotated through the proxy's A-records on + * every call. + *
  • A name supplied by an {@code AddressTranslator} ({@code SubnetAddressTranslator} returns + * one by default, under {@code resolve-addresses = false}) carries no such guarantee: it + * may cover several hosts. Spreading one node's connections across those would land the + * channels of a single {@code Node} on different servers, while routing, shard awareness + * and per-node metrics all attribute them to that one node. Such an endpoint keeps the + * resolver's order, so a pool converges on one address and the rest serve as fallback. + *
+ * + *

Only consulted for a node the driver has already identified. A contact point is spread + * across its addresses regardless, since they may well be different nodes and there is no node + * identity to preserve yet. + */ + default boolean addressesAreInterchangeable() { + return false; + } + + /** + * Whether two endpoints denote the same node and are indistinguishable to everything that + * reads one: same runtime type, same {@linkplain EndPoint#asMetricPrefix() metric identity}, same + * {@linkplain EndPoint#resolve() current address}. + * + *

Deliberately not {@link Object#equals}: {@code DefaultEndPoint#equals} resolves the + * unresolved side of a mixed comparison, which would put a blocking DNS lookup on the admin + * thread for every endpoint that is still a hostname — and contact points are now kept + * unresolved, so that is reachable (see issue #1006). It is also narrower than {@code equals} in + * one direction and wider in another, which is exactly what callers need: + * + *

    + *
  • Narrower: a pinned copy differs from its original only by the pin, and both {@code + * equals} and the metric identity ignore that by contract (above), so the {@code resolve()} + * comparison is what tells the two apart. + *
  • Wider: an unresolved hostname and the address it maps to compare equal under + * {@code DefaultEndPoint#equals} while their metric prefixes differ — the case a + * contact-point node hits when it adopts the endpoint built from its {@code system.local} + * row. + *
+ * + *

The class check keeps a node from staying on a plain fallback endpoint when a dynamic one + * ({@code ClientRoutesEndPoint}) with the same current address arrives. + * + *

{@code toString()} is not part of the test, even though {@code TaggingMetricIdGenerator} + * tags metrics with it rather than with the prefix. It is not stable across equal instances: + * {@code DefaultEndPoint} delegates it to {@code InetSocketAddress}, which renders {@code + * InetAddress}'s cached {@code hostName} field, and that field is populated the first time + * anything calls {@code getHostName()} — which {@code DefaultSslEngineFactory} does while + * building an engine, under the default {@code + * advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true}. Keying on it would report a + * difference for every node on every topology refresh. The cost of leaving it out: a tagging + * generator can keep reporting under an endpoint string the node no longer answers to, until + * something else changes the prefix. + */ + static boolean sameIdentity(@NonNull EndPoint first, @NonNull EndPoint second) { + return first.getClass() == second.getClass() + && first.asMetricPrefix().equals(second.asMetricPrefix()) + && Objects.equals(first.resolve(), second.resolve()); + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index d1ab8eec98d..18982f3b681 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -18,61 +18,140 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.shaded.guava.common.primitives.UnsignedBytes; import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.InetAddress; +import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Comparator; +import java.net.SocketAddress; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; -public class SniEndPoint implements EndPoint { - private static final AtomicInteger OFFSET = new AtomicInteger(); +public class SniEndPoint implements PinnableEndPoint { private final InetSocketAddress proxyAddress; private final String serverName; /** - * @param proxyAddress the address of the proxy. If it is {@linkplain - * InetSocketAddress#isUnresolved() unresolved}, each call to {@link #resolve()} will - * re-resolve it, fetch all of its A-records, and if there are more than 1 pick one in a - * round-robin fashion. + * The proxy IP this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}: a + * pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + + /** + * @param proxyAddress the address of the proxy. Stored {@linkplain + * InetSocketAddress#isUnresolved() unresolved}, whatever form it was supplied in, so that the + * driver expands a proxy hostname to all of its A-records at connection time and tries each + * of them — see {@link #storeUnresolved}. * @param serverName the SNI server name. In the context of Cloud, this is the string * representation of the host id. */ public SniEndPoint(InetSocketAddress proxyAddress, String serverName) { - this.proxyAddress = Objects.requireNonNull(proxyAddress, "SNI address cannot be null"); + this(proxyAddress, serverName, null); + } + + private SniEndPoint( + InetSocketAddress proxyAddress, + String serverName, + @Nullable InetSocketAddress pinnedAddress) { + this.proxyAddress = + storeUnresolved(Objects.requireNonNull(proxyAddress, "SNI address cannot be null")); this.serverName = Objects.requireNonNull(serverName, "SNI Server name cannot be null"); + this.pinnedAddress = pinnedAddress; + } + + /** + * Stores the proxy address unresolved, whatever form it arrived in. + * + *

{@link #resolve()} hands the stored address to the connection layer as-is, and only an + * unresolved one gets expanded and re-expanded there. A proxy hostname supplied already resolved + * would therefore stay bound to whichever single IP its lookup happened to return, for the life + * of the session: no spreading across the proxy's A-records, no fallback when that one IP stops + * answering, and no pick-up of a DNS change. That is a real possibility for a hostname handed to + * {@link + * com.datastax.oss.driver.api.core.session.SessionBuilder#withCloudProxyAddress(InetSocketAddress)}, + * because the ordinary {@code InetSocketAddress(String, int)} constructor resolves eagerly. + * ({@code CloudConfigFactory}, the usual path, already builds an unresolved address.) + * + *

An address that is already an IP literal is stored unresolved too, even though it has + * nothing to expand, because that is what makes this endpoint's identity stable. A + * resolved address's {@code getHostString()} is not fixed: it starts out as the IP literal and + * begins reporting the reverse-DNS name as soon as anything calls {@code getHostName()} on the + * underlying {@code InetAddress} — which {@code SniSslEngineFactory#newSslEngine} does, on this + * very instance, under the default {@code + * advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true}. Keying {@link #equals} and + * {@link #asMetricPrefix()} off a string that can change underneath them would move a node's + * metrics mid-session and make endpoints built before and after the first TLS handshake compare + * unequal. An unresolved address has no such field to fill in: its host string is fixed at + * construction, and {@code getHostName()} on it performs no lookup. SNI's reverse lookup still + * happens, on the {@linkplain #pinTo(SocketAddress) pinned} copy that carries the IP the channel + * actually reached. + * + *

What this cannot defend against is an instance the caller polluted before handing it over, + * i.e. called {@code getHostName()} on themselves. + * + *

Normalizing here rather than at the call site keeps every {@code SniEndPoint} built from the + * same proxy comparable — {@link #equals} keys on this field — and matches what this endpoint did + * before resolution moved to the connection layer, when it re-resolved the proxy hostname on + * every {@code resolve()} call. + */ + private static InetSocketAddress storeUnresolved(InetSocketAddress proxyAddress) { + return proxyAddress.isUnresolved() + ? proxyAddress + : InetSocketAddress.createUnresolved(proxyAddress.getHostString(), proxyAddress.getPort()); } public String getServerName() { return serverName; } + /** + * Returns the proxy address connections should be opened to. + * + *

Unpinned, this is the stored proxy address as-is — always unresolved (see {@link + * #storeUnresolved}), which {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} + * expands to every proxy A-record, trying each in turn — so a single unreachable proxy IP no + * longer fails the connection. Re-resolving here instead would block whichever event loop called + * us, and would bypass a custom Netty resolver. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} this returns that one proxy IP. That is what + * {@link com.datastax.oss.driver.internal.core.ssl.SniSslEngineFactory#newSslEngine} sees: it + * runs inside Netty's channel initializer, so it gets the exact IP the channel is connected to + * without a lookup on the event loop. + */ @NonNull @Override public InetSocketAddress resolve() { - try { - InetAddress[] aRecords = InetAddress.getAllByName(proxyAddress.getHostName()); - if (aRecords.length == 0) { - // Probably never happens, but the JDK docs don't explicitly say so - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName()); - } - // The order of the returned address is unspecified. Sort by IP to make sure we get a true - // round-robin - Arrays.sort(aRecords, IP_COMPARATOR); - int index = - (aRecords.length == 1) - ? 0 - : OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) % aRecords.length; - return new InetSocketAddress(aRecords[index], proxyAddress.getPort()); - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName(), e); + return pinnedAddress != null ? pinnedAddress : proxyAddress; + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + // Mirrors DefaultEndPoint and ClientRoutesEndPoint: an address this endpoint cannot hold in an + // InetSocketAddress field skips pinning rather than failing the connection, and so does an + // unresolved one. resolve() hands the proxy address over unresolved, and ChannelFactory passes + // it straight back when the user disabled the resolver or a custom one declines it; pinning + // that would freeze this endpoint on a name that must re-expand on every connect -- no address + // stability gained, and the proxy's A-record fallback silenced for good. + if (!(resolvedAddress instanceof InetSocketAddress) + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress)) { + return this; } + return new SniEndPoint(proxyAddress, serverName, (InetSocketAddress) resolvedAddress); + } + + /** + * {@inheritDoc} + * + *

{@code true}: the proxy routes by server name, so every one of its A-records reaches this + * same node, and connections may be spread across them. That restores what this endpoint did + * itself before resolution moved to the connection layer, when {@code resolve()} sorted the proxy + * A-records and rotated through them on every call. + */ + @Override + public boolean addressesAreInterchangeable() { + return true; } @Override @@ -94,10 +173,10 @@ public int hashCode() { @Override public String toString() { - // Note that this uses the original proxy address, so if there are multiple A-records it won't - // show which one was selected. If that turns out to be a problem for debugging, we might need - // to store the result of resolve() in Connection and log that instead of the endpoint. - return proxyAddress.toString() + ":" + serverName; + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which proxy IP a given + // connection landed on is in the channel's own toString(), which Netty builds from the actual + // remote address. + return proxyAddress + ":" + serverName; } @NonNull @@ -110,10 +189,4 @@ public String asMetricPrefix() { } return hostString.replace('.', '_') + ':' + proxyAddress.getPort() + '_' + serverName; } - - @SuppressWarnings("UnnecessaryLambda") - private static final Comparator IP_COMPARATOR = - (InetAddress address1, InetAddress address2) -> - UnsignedBytes.lexicographicalComparator() - .compare(address1.getAddress(), address2.getAddress()); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java index 1bb8e343d96..919fc26b8e0 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java @@ -141,4 +141,42 @@ public interface TopologyMonitor extends AsyncAutoCloseable { * {@link DefaultTopologyMonitor}) should override this method. */ default void resetColumnCaches() {} + + /** + * Whether this monitor re-resolves node addresses dynamically on every connection attempt (for + * example by re-resolving a proxy hostname each time), rather than relying on an endpoint address + * captured once at node-registration time. + * + *

When this returns {@code true}, the control connection's reconnection query plan must not + * append the original contact points as a DNS re-resolution fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}): the monitor + * already keeps addresses fresh, and appending raw contact points could resurrect nodes that the + * monitor has authoritatively removed. + * + *

The default implementation returns {@code false}, which is correct for {@link + * DefaultTopologyMonitor}: the peer nodes it registers hold a {@code DefaultEndPoint} built from + * the broadcast RPC address in {@code system.peers}, an already-resolved physical IP that never + * needs re-resolving. + * + *

Unless the configured {@code AddressTranslator} hands back a name -- {@code + * SubnetAddressTranslator} does, since its {@code resolve-addresses} option defaults to {@code + * false}. Such a peer endpoint is re-expanded per connection attempt by {@code + * ChannelFactory}, and if that name maps to more than one host, one {@code Node}'s connections + * can land on different ones while routing, shard awareness and per-node metrics all attribute + * them to that single node. The candidate loop keeps them in resolver order rather than shuffling + * for an identified node (see {@code ChannelFactory#shuffleAndLimit}), so a pool stays on one + * host in practice, but the driver has no way to verify the premise. That is a property of the + * translator's output, not of this monitor, so it does not change what this flag reports. + * + *

The connected node's own {@code EndPoint} is a different case again. It originates from the + * contact point the control connection used, and {@code ChannelFactory} binds it to the single + * address that connection reached (see {@code PinnableEndPoint}), so it does not re-expand + * on later connection attempts. Recovering from an address change for that node therefore depends + * on this flag being {@code false}, i.e. on the contact-point fallback described above. + * + *

Proxy-based monitors that re-resolve per call should override this to return {@code true}. + */ + default boolean reresolvesNodeAddresses() { + return false; + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java index 8905edb9192..7d408fb999b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.internal.core.util; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; +import com.datastax.oss.driver.shaded.guava.common.net.InetAddresses; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -56,4 +57,36 @@ public static Set extract(String address, boolean resolve) { return result; } } + + /** + * Whether {@code address} denotes a host name, as opposed to an IP address written out in + * literal form. + * + *

The distinction matters wherever a name is treated as something that can be resolved — and + * re-resolved — while a literal is taken as the final answer. Both forms can appear resolved or + * unresolved, so neither {@link InetSocketAddress#isUnresolved()} nor the presence of an {@link + * InetAddress} tells them apart. + * + *

Performs no lookup of any kind. + * + *

One known imprecision, on the unresolved branch: {@code InetAddresses.isInetAddress} rejects + * a zone suffix, so an unresolved address built over a scoped IPv6 literal (say {@code + * fe80::1%eth0}) is reported as a name. Harmless where this is used — the string still resolves + * to exactly that address, and re-attaching it as a "host name" is a no-op — and it does not + * affect the resolved branch, where {@code getHostString()} and {@code getHostAddress()} both + * carry the zone and so compare equal. + */ + public static boolean carriesName(InetSocketAddress address) { + String hostString = address.getHostString(); + if (hostString == null) { + return false; + } + // A resolved address is compared against the literal its own bytes produce, which is cheaper + // and + // stricter than parsing; an unresolved one has no bytes, so its string has to be parsed. + InetAddress ip = address.getAddress(); + return ip != null + ? !hostString.equals(ip.getHostAddress()) + : !InetAddresses.isInetAddress(hostString); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java index f31dd2861ed..cb853f2d801 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java @@ -18,11 +18,13 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import java.io.UncheckedIOException; +import io.netty.channel.local.LocalAddress; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -66,20 +68,23 @@ public void should_fallback_when_resolve_returns_null() throws UnknownHostExcept } @Test - public void should_wrap_io_exceptions_in_unchecked_io_exception() throws UnknownHostException { + public void should_return_the_route_address_unresolved() { + // The route hostname is handed over unresolved on purpose: ChannelFactory resolves it through + // Netty's AddressResolverGroup, so a custom resolver applies to client routes too and no DNS + // lookup runs on the admin event loop that connect() is called from. UUID hostId = UUID.randomUUID(); - when(topologyMonitor.resolve(hostId)).thenThrow(new UnknownHostException("no-such-host")); + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); ClientRoutesEndPoint ep = new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); - assertThatThrownBy(ep::resolve) - .isInstanceOf(UncheckedIOException.class) - .hasCauseInstanceOf(UnknownHostException.class); + assertThat(ep.resolve()).isSameAs(route); + assertThat(((InetSocketAddress) ep.resolve()).isUnresolved()).isTrue(); } @Test - public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownHostException { + public void should_reflect_route_changes_on_subsequent_resolve() { UUID hostId = UUID.randomUUID(); InetSocketAddress addr1 = new InetSocketAddress("127.0.0.1", 9042); InetSocketAddress addr2 = new InetSocketAddress("10.0.0.1", 9043); @@ -96,6 +101,67 @@ public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownH assertThat(ep.resolve()).isEqualTo(addr2); } + // ---- pinTo() ------------------------------------------------------------ + + @Test + public void pin_to_should_stop_consulting_the_topology_monitor() { + UUID hostId = UUID.randomUUID(); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + // No lookup at all -- that is the point: DefaultTopologyMonitor#savePort and the SSL factories + // read the channel's endpoint, and must not trigger a blocking re-resolution there. + verify(topologyMonitor, never()).resolve(hostId); + // Identity is keyed off the host id, so the pinned copy is still the same node. + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((ClientRoutesEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // resolve() hands out the route's hostname unresolved, and ChannelFactory passes an address + // straight back when the user disabled the resolver or a custom one declines it -- so that + // hostname can come back here. Pinning it would freeze the endpoint on a name that still + // re-expands on every connect, and silence the route lookup for good, since resolve() + // short-circuits once pinned. Both siblings return `this` for the same input. + UUID hostId = UUID.randomUUID(); + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); + ClientRoutesEndPoint endPoint = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + + assertThat(endPoint.pinTo(route)).isSameAs(endPoint); + // Still asking the monitor, which is what would have been lost. + assertThat(endPoint.resolve()).isEqualTo(route); + verify(topologyMonitor, atLeastOnce()).resolve(hostId); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // Mirror DefaultEndPoint: an address that cannot be held in an InetSocketAddress field (e.g. + // the local transport used by unit tests) skips pinning rather than failing the connection. + ClientRoutesEndPoint endPoint = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } + // ---- equals / hashCode -------------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java index a1ba4617ef5..e6ab0895034 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java @@ -28,6 +28,8 @@ import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Metadata; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; import com.datastax.oss.driver.internal.core.channel.DriverChannel; @@ -66,6 +68,8 @@ public class ClientRoutesTopologyMonitorTest { @Mock private ControlConnection controlConnection; @Mock private DriverConfig driverConfig; @Mock private DriverExecutionProfile defaultProfile; + @Mock private MetadataManager metadataManager; + @Mock private Metadata metadata; private TestableClientRoutesTopologyMonitor handler; @@ -194,14 +198,21 @@ public void should_throw_after_close() { } @Test - public void should_throw_for_unresolvable_hostname() { + public void should_not_look_up_the_route_hostname() { UUID hostId = UUID.randomUUID(); - // Use a hostname guaranteed not to resolve + // A hostname guaranteed not to resolve: this must still succeed, because resolve() is a pure + // in-memory cache lookup that hands the name over unresolved. ChannelFactory resolves it later + // through Netty's AddressResolverGroup, so a custom resolver applies to client routes too and + // nothing blocks the admin event loop here. handler.setRoutes( ImmutableMap.of( hostId, new ClientRouteRecord(hostId, "this.host.does.not.exist.invalid", 9042))); - assertThatThrownBy(() -> handler.resolve(hostId)).isInstanceOf(UnknownHostException.class); + InetSocketAddress result = handler.resolve(hostId); + + assertThat(result.isUnresolved()).isTrue(); + assertThat(result.getHostString()).isEqualTo("this.host.does.not.exist.invalid"); + assertThat(result.getPort()).isEqualTo(9042); } @Test @@ -220,6 +231,57 @@ public void should_refresh_updates_routes() throws UnknownHostException { assertThat(handler.resolve(hostId2)).isNotNull(); } + // ---- reresolvesNodeAddresses() ------------------------------------------- + + @Test + public void should_reresolve_when_all_known_nodes_have_client_routes() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + handler.setRoutes( + ImmutableMap.of( + hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042), + hostId2, new ClientRouteRecord(hostId2, "127.0.0.2", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isTrue(); + } + + @Test + public void should_not_reresolve_when_a_known_node_has_no_client_route() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + // Only node1 has a live client route; node2 would fall back to a static endpoint. + handler.setRoutes(ImmutableMap.of(hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isFalse(); + } + + @Test + public void should_reresolve_when_no_nodes_known_yet() { + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(Collections.emptyMap()); + + assertThat(handler.reresolvesNodeAddresses()).isTrue(); + } + // ---- Merge behavior tests ----------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index 7da8fb39415..9c22b93e1e8 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -20,6 +20,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import io.netty.channel.local.LocalAddress; import java.net.InetSocketAddress; import org.junit.Test; @@ -57,4 +59,140 @@ public void should_reject_null_address() { .isInstanceOf(NullPointerException.class) .hasMessage("address can't be null"); } + + @Test + public void resolve_returns_already_resolved_address_as_is() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + InetSocketAddress resolved = endPoint.resolve(); + assertThat(resolved.isUnresolved()).isFalse(); + assertThat(resolved.getHostString()).isEqualTo("127.0.0.1"); + } + + @Test + public void resolve_passes_unresolved_hostname_through_without_looking_it_up() { + // This endpoint does NOT resolve hostnames itself. It hands the unresolved address to + // ChannelFactory, which expands it through Netty's AddressResolverGroup so that a custom + // resolver installed via NettyOptions#afterBootstrapInitialized still applies -- a direct + // InetAddress.getAllByName() call here would bypass it, and would block the admin event loop + // that connect() runs on. "localhost" would resolve fine, so this assertion is only meaningful + // because we check the address comes back *unresolved*. + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + + InetSocketAddress resolved = endPoint.resolve(); + + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo("localhost"); + assertThat(resolved.getPort()).isEqualTo(9042); + } + + @Test + public void resolve_does_not_throw_for_unresolvable_hostname() { + // No lookup happens, so an unresolvable name is not an error at this level: the connect attempt + // fails later with a descriptive error instead. + DefaultEndPoint endPoint = + new DefaultEndPoint( + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042)); + + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); + } + + @Test + public void pin_to_should_override_resolution_but_preserve_identity() { + InetSocketAddress hostname = InetSocketAddress.createUnresolved("test.com", 9042); + DefaultEndPoint original = new DefaultEndPoint(hostname); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + // Resolution now yields exactly the pinned address... + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + // ...but the copy still denotes the same node, and metric names must not change depending on + // which IP a connection happened to land on -- including through toString(), which is what + // TaggingMetricIdGenerator tags node metrics with, and nodes do adopt pinned copies. + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); + assertThat(pinned).isEqualTo(original); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + // Equality has to hold in both directions: endpoints are used as set and map keys. + assertThat(original).isEqualTo(pinned); + // The original is untouched. + assertThat(original.resolve()).isEqualTo(hostname); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + DefaultEndPoint original = + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.com", 9042)); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((DefaultEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_return_same_instance_when_address_is_already_the_endpoints_own() { + // An already-resolved endpoint expands to exactly one candidate -- itself -- so ChannelFactory + // pins it to the address it already holds. That copy would be indistinguishable from the + // original in every respect, so there is no point allocating it. Every node discovered from the + // peers rows takes this path. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + DefaultEndPoint endPoint = new DefaultEndPoint(resolved); + + assertThat(endPoint.pinTo(new InetSocketAddress("127.0.0.1", 9042))).isSameAs(endPoint); + assertThat(endPoint.toString()).isEqualTo(resolved.toString()); + } + + @Test + public void pin_to_should_reject_null_address() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThatThrownBy(() -> endPoint.pinTo(null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("resolvedAddress can't be null"); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // ChannelFactory pins whatever address it connected to; a non-Inet one (e.g. the local + // transport + // used by unit tests) cannot be held in an InetSocketAddress field, so pinning is skipped + // rather + // than failing the connection. + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // resolveCandidates() hands the original address straight through when the user disabled + // Netty's + // resolver, when the resolver does not support the address, or when it reports it as already + // resolved. Pinning a name would freeze resolve() on something that must re-expand on every + // connect -- no address stability gained, and the endpoint's own source silenced. The same + // guard + // is in SniEndPoint and ClientRoutesEndPoint. + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.com", 9042)); + + assertThat(endPoint.pinTo(InetSocketAddress.createUnresolved("db.example.com", 9042))) + .isSameAs(endPoint); + } + + @Test + public void should_still_compare_an_unresolved_address_against_a_resolved_one() { + // Kept working for Metadata#findNode, whose caller may hold either form -- but it costs a DNS + // lookup on the calling thread, only compares the first address the name maps to, and does not + // agree with hashCode(). equals() warns once per JVM about it; that latch is static, so whether + // this test is the one that trips it depends on test ordering and is deliberately not asserted. + // See https://github.com/scylladb/java-driver/issues/1006. + DefaultEndPoint unresolved = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + DefaultEndPoint resolved = new DefaultEndPoint(new InetSocketAddress("localhost", 9042)); + + assertThat(unresolved).isEqualTo(resolved); + assertThat(resolved).isEqualTo(unresolved); + // ... while hashing differently, which is exactly the inconsistency the issue tracks. + assertThat(unresolved.hashCode()).isNotEqualTo(resolved.hashCode()); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java new file mode 100644 index 00000000000..416821798d9 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import org.junit.Test; + +public class SniEndPointTest { + + @Test + public void resolve_returns_the_proxy_address_as_is_without_looking_it_up() { + // The proxy address is a hostname (that is how CloudConfigFactory builds it) and this endpoint + // must not resolve it: ChannelFactory expands it through Netty's AddressResolverGroup, which is + // what makes a custom resolver apply to the SNI proxy too, and what keeps resolve() safe to + // call + // from an event loop -- SniSslEngineFactory#newSslEngine does exactly that. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.com", 9042); + SniEndPoint endPoint = new SniEndPoint(proxy, "test-server-name"); + + assertThat(endPoint.resolve()).isSameAs(proxy); + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + } + + @Test + public void should_keep_a_resolved_proxy_hostname_unresolved() { + // InetSocketAddress(String, int) resolves eagerly, so a hostname passed to + // withCloudProxyAddress() arrives here already bound to one of its IPs. Storing it that way + // would freeze every Cloud connection on that IP for the life of the session: resolve() hands + // the stored address straight to the connection layer, which only expands unresolved ones. + InetSocketAddress resolvedProxy = new InetSocketAddress("localhost", 9042); + assertThat(resolvedProxy.isUnresolved()).isFalse(); + + SniEndPoint endPoint = new SniEndPoint(resolvedProxy, "test-server-name"); + + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + assertThat(endPoint.resolve().getHostString()).isEqualTo("localhost"); + assertThat(endPoint.resolve().getPort()).isEqualTo(9042); + // Normalization is unconditional, so endpoints built from either form of the same proxy still + // denote the same node -- equals() keys on the stored address. + assertThat(endPoint) + .isEqualTo( + new SniEndPoint( + InetSocketAddress.createUnresolved("localhost", 9042), "test-server-name")); + // The metric prefix is unaffected either way: it was already built from the host string. + assertThat(endPoint.asMetricPrefix()).isEqualTo("localhost:9042_test-server-name"); + } + + @Test + public void should_store_a_proxy_given_as_an_ip_address_unresolved_too() { + // An IP literal has nothing to expand, but it is stored unresolved all the same, because that + // is what makes this endpoint's identity immune to drift: see the test below. + InetSocketAddress ipProxy = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint endPoint = new SniEndPoint(ipProxy, "test-server-name"); + + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + assertThat(endPoint.resolve().getHostString()).isEqualTo("127.0.0.1"); + assertThat(endPoint.resolve().getPort()).isEqualTo(9042); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + } + + @Test + public void identity_should_not_drift_when_the_source_address_grows_a_reverse_dns_name() { + // SniSslEngineFactory#newSslEngine calls getHostName() on the address resolve() hands it, under + // the default allow-dns-reverse-lookup-san = true. On a *resolved* address that caches the + // reverse-DNS name on the underlying InetAddress, and getHostString() reports it from then on. + // An endpoint keyed on such an instance would silently change its metric prefix mid-session, + // moving a node's metrics; storing the address unresolved removes the field that changes. + InetSocketAddress ipProxy = new InetSocketAddress("127.0.0.1", 9042); + SniEndPoint endPoint = new SniEndPoint(ipProxy, "test-server-name"); + InetSocketAddress storedBefore = endPoint.resolve(); + + // What the SSL engine factory does. 127.0.0.1 has a PTR record on essentially every machine, so + // this really does change the source instance's host string. + ipProxy.getHostName(); + assertThat(ipProxy.getHostString()).isNotEqualTo("127.0.0.1"); + + assertThat(endPoint.resolve()).isEqualTo(storedBefore); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + assertThat(endPoint).isEqualTo(new SniEndPoint(storedBefore, "test-server-name")); + + // And the driver no longer pollutes the address it holds in the first place: getHostName() on + // an + // unresolved address returns its stored host string without looking anything up, so the call + // above cannot happen to *this* instance. (What the SSL engine actually sees is the pinned + // copy, + // whose address is excluded from equals() and asMetricPrefix() by contract.) + assertThat(endPoint.resolve().getHostName()).isEqualTo("127.0.0.1"); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + } + + @Test + public void resolve_does_not_throw_for_unresolvable_proxy_hostname() { + // No lookup happens here, so an unresolvable name only fails later, at connect time. + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042), + "test-server-name"); + + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); + } + + @Test + public void pin_to_should_make_resolve_return_the_connected_proxy_ip_and_preserve_identity() { + // Pinning is what lets SniSslEngineFactory#newSslEngine -- which runs inside Netty's channel + // initializer -- see the very proxy IP the channel is connected to, rather than the hostname or + // another A-record. + SniEndPoint original = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + assertThat(pinned.resolve().isUnresolved()).isFalse(); + // The original is untouched. + assertThat(original.resolve().isUnresolved()).isTrue(); + + // The pinned copy still denotes the same node, down to every string it is identified by: the + // tagging MetricIdGenerator tags node metrics with the endpoint's toString(), and nodes do + // adopt + // pinned copies. + assertThat(pinned.getServerName()).isEqualTo(original.getServerName()); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + SniEndPoint original = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + assertThat(pinned.pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // ChannelFactory hands the address straight back when the user disabled Netty's resolver or a + // custom one declines it. Pinning that would freeze resolve() on a name that must re-expand on + // every connect, silencing the proxy's A-record fallback for good. + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + + assertThat(endPoint.pinTo(InetSocketAddress.createUnresolved("proxy.example.com", 9042))) + .isSameAs(endPoint); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + + assertThat(endPoint.pinTo(new LocalAddress("test"))).isSameAs(endPoint); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java new file mode 100644 index 00000000000..f48d70ef838 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import org.junit.Test; + +public class AddressUtilsTest { + + @Test + public void should_recognize_a_hostname_whether_resolved_or_not() { + assertThat( + AddressUtils.carriesName(InetSocketAddress.createUnresolved("host.example.com", 9042))) + .isTrue(); + // Eagerly resolved by the constructor, but still a name. + assertThat(AddressUtils.carriesName(new InetSocketAddress("localhost", 9042))).isTrue(); + } + + @Test + public void should_not_mistake_an_ip_literal_for_a_hostname() throws Exception { + assertThat(AddressUtils.carriesName(new InetSocketAddress("127.0.0.1", 9042))).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("127.0.0.1", 9042))) + .isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("::1", 9042))).isFalse(); + // Built from raw bytes, so it carries no name at all and getHostString() falls back to the + // literal -- without triggering the reverse lookup that getHostName() would. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042))) + .isFalse(); + } + + @Test + public void should_report_an_explicitly_named_address_as_a_name() throws Exception { + // A resolver may label its results with a name of its own; that is still a name. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress( + InetAddress.getByAddress("cname.example.com", new byte[] {10, 0, 0, 1}), 9042))) + .isTrue(); + } +} From f5a5499d7b1070741c42e9c75c42e2eade305bc9 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:15:52 +0200 Subject: [PATCH 4/9] feat: try every address a hostname resolves to when connecting (DRIVER-201) This is the fix for DRIVER-201. When a contact point or a node address is a hostname that maps to several IPs, the driver used to try only the first one and raise AllNodesFailedException if it was unreachable, even though the hostname also resolved to healthy addresses. Resolution is a connection-layer concern. ChannelFactory.connect() is now the single place that turns "the address this node is known by" into "the addresses to actually try": EndPoint.resolve() yields one address and does no lookup, so it stays safe to call from an event loop; ChannelFactory expands it through the bootstrap's Netty AddressResolverGroup; the candidates are tried in sequence until one connects; and the endpoint is pinned to the address that won, so the channel carries the address it is really on. Expansion always goes through the configured resolver, mirroring Netty's own doResolveAndConnect0 short-circuit (no group, !isSupported, isResolved) rather than pre-filtering on isUnresolved(). Both isSupported() and isResolved() are overridable, so a redirecting custom resolver keeps its say over addresses that merely look resolved. The bootstrap is built once per connect() and cloned per attempt, with the clone's resolver disabled: Bootstrap.clone() carries the resolver over, so an enabled clone would resolve each candidate a second time -- through resolve(), singular -- and a redirecting resolver would collapse every candidate onto its first answer, silently killing the fallback. Details that took a round each to get right: - The queried hostname is re-attached to resolver-returned addresses, centrally rather than per endpoint, so TLS sees the name the user configured instead of an IP or a CNAME label. Scoped IPv6 keeps its zone via the numeric Inet6Address.getByAddress overload; the NetworkInterface one re-derives the scope and throws when the interface has no address of the same local type. - One EventLoop is chosen per connect() and shared by resolution and every clone(eventLoop), instead of letting Bootstrap.connect() advance the chooser a second time and land channels on half the loops. - Candidates are shuffled per connect and truncated to the new advanced.connection.max-candidate-addresses option (default 5). The shuffle spreads load and varies the starting address between attempts with no per-name state to maintain; the cap bounds what one attempt can cost -- each address tried is a full connect plus handshake, and with wrong credentials a rejected login -- while successive attempts sample fresh random subsets, so no address is permanently out of reach. - Protocol-version rejection is terminal only for a node whose host id is known. The addresses of an unidentified endpoint may belong to different nodes, and collapsing a contact-point hostname into one Node must not lose the query-plan advance that resolve-contact-points=true used to provide. An authentication failure is never terminal, even for an identified node: with a multi-record name it may be specific to the address (a stale record pointing at a foreign cluster fails at AUTH, which runs before the cluster-name check), a single address's failure must not write off the endpoint, and the candidate cap is what bounds the cost of genuinely wrong credentials. - Failures from earlier candidates are attached to the final error as suppressed exceptions -- with the surfaced one chosen by how callers classify errors rather than by position -- and negotiation history is scoped per candidate address. - Every resolver and Netty callback completes the connect future on failure. connect() has no timeout at the resolution stage, so an unguarded throw would hang the caller for good. afterBootstrapInitialized() now runs once per logical connection rather than once per attempt, and sees the bootstrap before the driver's handler is installed; a handler set by the hook is overwritten, with a one-time warning. Co-Authored-By: Claude Opus 5 (1M context) Co-Authored-By: Claude Fable 5 --- .../api/core/config/DefaultDriverOption.java | 7 + .../driver/api/core/config/OptionsMap.java | 1 + .../api/core/config/TypedDriverOption.java | 7 + .../internal/core/channel/ChannelFactory.java | 1066 +++++++++++++++-- .../internal/core/context/NettyOptions.java | 16 +- .../core/control/ControlConnection.java | 53 +- core/src/main/resources/reference.conf | 17 + .../ChannelFactoryBootstrapHookTest.java | 71 ++ .../ChannelFactoryMultiAddressTest.java | 816 +++++++++++++ .../ChannelFactoryNettyResolverTest.java | 447 +++++++ .../ChannelFactoryPinnedEndPointTest.java | 233 ++++ ...ChannelFactoryProtocolNegotiationTest.java | 217 ++++ .../core/channel/ChannelFactoryTestBase.java | 58 + .../channel/TestAddressResolverGroup.java | 130 ++ .../core/control/ControlConnectionTest.java | 56 + 15 files changed, 3094 insertions(+), 101 deletions(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index 3be41810660..f026558171c 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -135,6 +135,13 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: int */ CONNECTION_MAX_ORPHAN_REQUESTS("advanced.connection.max-orphan-requests"), + /** + * The maximum number of addresses a single connection attempt will try, when the endpoint it + * connects to is a DNS name that resolves to several addresses. + * + *

Value-type: int + */ + CONNECTION_MAX_CANDIDATE_ADDRESSES("advanced.connection.max-candidate-addresses"), /** * Whether to log non-fatal errors when the driver tries to open a new connection. * diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index 293231347fe..0d86d7b1c62 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -279,6 +279,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.CONNECTION_POOL_INIT_BATCH_SIZE, 0); map.put(TypedDriverOption.CONNECTION_MAX_REQUESTS, 1024); map.put(TypedDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, 256); + map.put(TypedDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, 5); map.put(TypedDriverOption.CONNECTION_WARN_INIT_ERROR, true); map.put(TypedDriverOption.CONNECTION_ADVANCED_SHARD_AWARENESS_ENABLED, true); map.put(TypedDriverOption.ADVANCED_SHARD_AWARENESS_PORT_LOW, 10000); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index b0a5e788c67..8a8f5512644 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -172,6 +172,13 @@ public String toString() { public static final TypedDriverOption CONNECTION_MAX_ORPHAN_REQUESTS = new TypedDriverOption<>( DefaultDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, GenericType.INTEGER); + /** + * The maximum number of addresses a single connection attempt will try, when the endpoint it + * connects to is a DNS name that resolves to several addresses. + */ + public static final TypedDriverOption CONNECTION_MAX_CANDIDATE_ADDRESSES = + new TypedDriverOption<>( + DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, GenericType.INTEGER); /** Whether to log non-fatal errors when the driver tries to open a new connection. */ public static final TypedDriverOption CONNECTION_WARN_INIT_ERROR = new TypedDriverOption<>(DefaultDriverOption.CONNECTION_WARN_INIT_ERROR, GenericType.BOOLEAN); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 35190afa3f4..c37245103b1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -24,8 +24,10 @@ package com.datastax.oss.driver.internal.core.channel; import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.InvalidKeyspaceException; import com.datastax.oss.driver.api.core.ProtocolVersion; import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; @@ -39,11 +41,14 @@ import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.SessionMetricUpdater; import com.datastax.oss.driver.internal.core.protocol.FrameDecoder; import com.datastax.oss.driver.internal.core.protocol.FrameEncoder; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; @@ -54,14 +59,28 @@ import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoop; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.Future; import java.io.IOException; +import java.net.Inet6Address; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.NetworkInterface; import java.net.ServerSocket; import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Random; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; @@ -127,6 +146,19 @@ public static int effectiveMaxOrphanRequests( private final String logPrefix; protected final InternalDriverContext context; + /** + * Guards the one-time warning in {@link #newBootstrap()}. Per factory rather than per JVM: what + * it reports is a property of this session's {@link NettyOptions}, and the message names the + * session, so a JVM-wide latch would report the first offender and silence every one after it. + */ + private final AtomicBoolean loggedHandlerWarning = new AtomicBoolean(); + + /** + * Randomizes the order in which a name's expanded addresses are tried (see {@link + * #shuffleAndLimit}). Injectable so tests can seed it and observe a deterministic order. + */ + @VisibleForTesting Random random = new Random(); + /** either set from the configuration, or null and will be negotiated */ @VisibleForTesting volatile ProtocolVersion protocolVersion; @@ -145,6 +177,7 @@ public ChannelFactory(InternalDriverContext context) { this.context = context; DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); + if (defaultConfig.isDefined(DefaultDriverOption.PROTOCOL_VERSION)) { String versionName = defaultConfig.getString(DefaultDriverOption.PROTOCOL_VERSION); this.protocolVersion = context.getProtocolVersionRegistry().fromName(versionName); @@ -181,7 +214,7 @@ public CompletionStage connect(Node node, DriverChannelOptions op } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater); + return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater, isIdentified(node)); } public CompletionStage connect( @@ -192,7 +225,31 @@ public CompletionStage connect( } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), node.getShardingInfo(), shardId, options, nodeMetricUpdater); + return connect( + node.getEndPoint(), + node.getShardingInfo(), + shardId, + options, + nodeMetricUpdater, + isIdentified(node)); + } + + /** + * Whether we know which node we are connecting to, as opposed to merely which address to + * try. {@link #tryNextCandidate} needs the distinction because an unidentified contact-point name + * may expand to addresses of different nodes, while every address of an identified node is + * that same node. + * + *

{@link Node#getHostId()} is null exactly for a contact point, and stays null for the life of + * that instance: {@code MetadataManager.registerNode} mints a fresh {@code DefaultNode} from each + * {@code system.local}/{@code system.peers} row rather than back-filling the contact point it was + * reached through, and those ephemeral contact-point nodes are never added to metadata. So this + * is not a state a contact point grows out of once the driver has read host ids -- the driver + * simply stops using that instance for anything except the reconnection fallback, which keeps + * handing it back. + */ + private static boolean isIdentified(Node node) { + return node.getHostId() != null; } @VisibleForTesting @@ -202,11 +259,23 @@ CompletionStage connect( Integer shardId, DriverChannelOptions options, NodeMetricUpdater nodeMetricUpdater) { + // A bare endpoint carries no host id, so this matches the contact-point case (see + // isIdentified()). + return connect(endPoint, shardingInfo, shardId, options, nodeMetricUpdater, false); + } + + @VisibleForTesting + CompletionStage connect( + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + boolean nodeIsIdentified) { CompletableFuture resultFuture = new CompletableFuture<>(); ProtocolVersion currentVersion; boolean isNegotiating; - List attemptedVersions = new CopyOnWriteArrayList<>(); if (this.protocolVersion != null) { currentVersion = protocolVersion; isNegotiating = false; @@ -223,7 +292,7 @@ CompletionStage connect( nodeMetricUpdater, currentVersion, isNegotiating, - attemptedVersions, + nodeIsIdentified, resultFuture); return resultFuture; } @@ -236,119 +305,936 @@ private void connect( NodeMetricUpdater nodeMetricUpdater, ProtocolVersion currentVersion, boolean isNegotiating, - List attemptedVersions, + boolean nodeIsIdentified, CompletableFuture resultFuture) { - SocketAddress resolvedAddress; + // Built once per connect() rather than once per candidate: it is the only handle on the Netty + // AddressResolverGroup (see resolveCandidates()), and it means the user's + // afterBootstrapInitialized() hook runs once per logical connection instead of once per address + // attempt. Each attempt gets its own clone() with its own handler. + // + // The event loop is likewise picked once per connect() and shared by name resolution and the + // channel itself (the per-attempt clones are bound to it, see connectToAddress()). Advancing + // the group's round-robin chooser exactly once per connect keeps channels evenly distributed: + // taking one loop for resolution and letting Bootstrap.connect() take another would advance + // the chooser twice per connect, parking all channels on half the loops with the default + // power-of-two chooser. It also mirrors what Netty itself does with an unresolved address: + // Bootstrap resolves on the connecting channel's own event loop. + Bootstrap baseBootstrap; + EventLoop eventLoop; try { - resolvedAddress = endPoint.resolve(); + baseBootstrap = newBootstrap(); + eventLoop = context.getNettyOptions().ioEventLoopGroup().next(); } catch (Exception e) { resultFuture.completeExceptionally(e); return; } - NettyOptions nettyOptions = context.getNettyOptions(); + // EndPoint.resolve() is contractually non-blocking and performs no name resolution, so it is + // safe to call here even though connect() runs on the admin event loop for control-connection + // reconnects. Everything a name needs to become connectable happens in resolveCandidates(). + SocketAddress address; + try { + address = endPoint.resolve(); + } catch (Exception e) { + resultFuture.completeExceptionally(e); + return; + } + if (address == null) { + // EndPoint.resolve() is contractually non-null; fail fast instead of NPE-ing inside an + // event-loop task later, which would leave resultFuture hanging (see resolveCandidates()). + resultFuture.completeExceptionally( + new IllegalArgumentException("EndPoint.resolve() returned null: " + endPoint)); + return; + } + + resolveCandidates( + baseBootstrap, address, eventLoop, spreadAcrossAddresses(endPoint, nodeIsIdentified)) + .whenComplete( + (candidates, error) -> { + if (error != null) { + Throwable cause = + (error instanceof CompletionException && error.getCause() != null) + ? error.getCause() + : error; + resultFuture.completeExceptionally(cause); + return; + } + tryNextCandidate( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + nodeIsIdentified, + resultFuture, + candidates, + 0, + new ArrayList<>()); + }); + } + /** + * Builds the {@link Bootstrap} shared by every connection attempt of a single {@code connect()} + * call, including the user's {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} hook. Per + * attempt, {@link #connectToAddress} takes a {@link + * Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it bound to the event loop the connect + * picked, and installs its own handler; the copy carries the resolver configuration over. The + * base bootstrap itself keeps the full I/O group, so the hook observes the same group as always. + */ + private Bootstrap newBootstrap() { + NettyOptions nettyOptions = context.getNettyOptions(); Bootstrap bootstrap = new Bootstrap() .group(nettyOptions.ioEventLoopGroup()) .channel(nettyOptions.channelClass()) - .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()) - .handler( - initializer(endPoint, currentVersion, options, nodeMetricUpdater, resultFuture)); - + .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()); nettyOptions.afterBootstrapInitialized(bootstrap); + if (bootstrap.config().handler() != null && loggedHandlerWarning.compareAndSet(false, true)) { + LOG.warn( + "[{}] NettyOptions.afterBootstrapInitialized() installed a channel handler on the" + + " bootstrap; it will be replaced by the driver's own handler. Use" + + " NettyOptions.afterChannelInitialized() to customize the pipeline instead.", + logPrefix); + } + return bootstrap; + } + + /** + * Turns the address an {@link EndPoint} denotes into the concrete, connectable addresses to try, + * expanding it to all the addresses it maps to when it is a name. + * + *

Expansion goes through the bootstrap's Netty {@link AddressResolverGroup} rather than a + * direct {@code InetAddress.getAllByName()} call, so a custom resolver installed via {@link + * NettyOptions#afterBootstrapInitialized(Bootstrap)} is honoured — that is the resolver an + * unresolved address would have reached had it been handed straight to {@code + * Bootstrap.connect()}, as it was before multi-address support. This is also why endpoints are + * forbidden from resolving names themselves (see {@link EndPoint#resolve()}): doing it here is + * the only way to keep that configuration point working, and the only way to keep {@code + * resolve()} non-blocking. + * + *

Whether an address needs resolving at all is the resolver's decision, not ours: exactly as + * in {@code Bootstrap#doResolveAndConnect0}, the address is passed through untouched only when + * the resolver says it does not {@linkplain AddressResolver#isSupported support} it (e.g. {@link + * io.netty.channel.local.LocalAddress}) or that it {@linkplain AddressResolver#isResolved is + * already resolved}. Both are overridable, and a custom resolver may well report an + * already-resolved address as unresolved in order to redirect it — Netty consulted it either way, + * so a pre-check here on {@code InetSocketAddress#isUnresolved()} would silently take that + * configuration point away for every connect to an already-resolved node, which is to say for + * almost every connect. A null group means the user called {@link Bootstrap#disableResolver()}, + * which is likewise respected. + * + *

Note that with Netty's default resolver the lookup blocks the event loop it runs on, + * because {@code DefaultNameResolver} performs {@code InetAddress.getAllByName()} inline. That is + * the pre-existing behaviour of handing an unresolved address to {@code Bootstrap.connect()}, and + * it is an I/O loop, never the admin loop that {@code connect()} is called from. Deployments that + * need non-blocking resolution can now install {@code DnsAddressResolverGroup} and have it take + * effect. + */ + private CompletionStage> resolveCandidates( + Bootstrap bootstrap, + SocketAddress address, + EventLoop eventLoop, + boolean spreadAcrossAddresses) { + + AddressResolverGroup resolverGroup = bootstrap.config().resolver(); + if (resolverGroup == null) { + // Bootstrap.disableResolver(): the user wants the address passed through as-is, which only + // works if it is usable as-is. + IllegalStateException unusable = + unusableWithoutResolution(address, "the bootstrap has name resolution disabled"); + return (unusable != null) + ? CompletableFutures.failedFuture(unusable) + : CompletableFuture.completedFuture(Collections.singletonList(address)); + } - ChannelFuture connectFuture; - if (shardId == null || shardingInfo == null) { - if (shardId != null) { + // The supplied event loop is the same one the channel will be registered on (see connect()), + // which is what Netty itself does with an unresolved address: Bootstrap resolves on the + // connecting channel's own event loop. Its transport also matches the channel class, which + // matters because DnsAddressResolverGroup registers a datagram channel on the executor it + // resolves for. + CompletableFuture> result = new CompletableFuture<>(); + // Every path below must complete `result`: nothing at this stage has a timeout, so a task or + // listener that dies with the future still pending (Netty swallows their throwables, it only + // logs them) would hang the connect attempt -- and with it control-connection init or a pool + // reconnect -- forever. Hence the blanket catches around the task body, the listener body, and + // the execute() call itself (which throws RejectedExecutionException while shutting down). + try { + eventLoop.execute( + () -> { + try { + AddressResolver resolver = + resolverGroup.getResolver(eventLoop); + if (!resolver.isSupported(address) || resolver.isResolved(address)) { + // Nothing for the resolver to do; same short-circuit as + // Bootstrap#doResolveAndConnect0. An address the resolver declines is in the same + // position as one with no resolver at all, so it gets the same check; an + // already-resolved one is usable by definition and passes straight through. + IllegalStateException unusable = + unusableWithoutResolution( + address, "the configured resolver does not support this address"); + if (unusable != null) { + result.completeExceptionally(unusable); + } else { + result.complete(Collections.singletonList(address)); + } + return; + } + resolver + .resolveAll(address) + .addListener( + (Future> future) -> { + try { + if (!future.isSuccess()) { + result.completeExceptionally(future.cause()); + return; + } + @SuppressWarnings("unchecked") + List addresses = + (List) future.getNow(); + if (addresses == null || addresses.isEmpty()) { + result.completeExceptionally( + new IllegalStateException( + "Resolver returned no address for " + address)); + return; + } + List connectable = + dropUnresolved(address, reattachHostnames(address, addresses)); + if (connectable.isEmpty()) { + result.completeExceptionally( + new IllegalStateException( + String.format( + "Cannot connect to %s: the configured resolver (%s) " + + "expanded it to %d address(es) and every one of them " + + "is still unresolved, so nothing will resolve them.", + address, resolver.getClass().getName(), addresses.size()))); + return; + } + result.complete(shuffleAndLimit(connectable, spreadAcrossAddresses)); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + } catch (Throwable t) { + result.completeExceptionally(t); + } + return result; + } + + /** + * The failure to report when {@link #resolveCandidates} is about to pass an address through + * without resolving it, or {@code null} if passing it through is fine. + * + *

{@link #connectToAddress} hands the candidate to a bootstrap clone with {@link + * Bootstrap#disableResolver()}, so an address that is still unresolved by the time it gets there + * cannot connect: Netty raises {@code UnresolvedAddressException} from inside {@code doConnect}, + * naming neither the address nor the reason nothing resolved it. That is a hard failure of every + * connection attempt for the whole session, and it is worth a message that says which endpoint + * and which configuration produced it -- the endpoints most likely to hit it (SNI, client routes) + * hand out unresolved addresses by design, and contact-point hostnames are now always kept + * unresolved. + * + *

Deliberately not a general {@code isUnresolved()} pre-check on every path: see {@link + * #resolveCandidates}'s javadoc for why an address the resolver merely declines to touch must + * still go through. This fires only where nothing downstream will resolve it either -- as does + * {@link #dropUnresolved}, which applies the same reasoning to what {@code resolveAll} returns. + */ + private static IllegalStateException unusableWithoutResolution( + SocketAddress address, String why) { + if (!(address instanceof InetSocketAddress) || !((InetSocketAddress) address).isUnresolved()) { + return null; + } + return new IllegalStateException( + String.format( + "Cannot connect to %s: it is an unresolved address and %s, so nothing will resolve it. " + + "Either remove Bootstrap.disableResolver() from " + + "NettyOptions.afterBootstrapInitialized(), or supply an already-resolved address.", + address, why)); + } + + /** Applies {@link #reattachHostname} to every expanded candidate. */ + private static List reattachHostnames( + SocketAddress original, List candidates) { + List result = new ArrayList<>(candidates.size()); + for (SocketAddress candidate : candidates) { + result.add(reattachHostname(original, candidate)); + } + return result; + } + + /** + * Drops the candidates a resolver returned still unresolved, keeping the order of the rest. + * + *

{@code resolveAll} is contracted to return resolved addresses, but a custom resolver that + * rewrites what it is given -- which {@link #resolveCandidates} deliberately supports -- may hand + * back one that is not. Such a candidate cannot connect: {@link #connectToAddress} uses a + * bootstrap clone with {@link Bootstrap#disableResolver()}, so nothing downstream will resolve it + * either, and Netty raises {@code UnresolvedAddressException} from inside {@code doConnect}. This + * is the same reasoning as {@link #unusableWithoutResolution}, applied where the addresses come + * from the resolver itself; dropping them here rather than after the cap means the cap counts + * only addresses that can actually be tried. The caller reports the case where nothing is left, + * which is the one that fails every connection attempt for the whole session. + */ + private List dropUnresolved( + SocketAddress original, List candidates) { + List result = new ArrayList<>(candidates.size()); + for (SocketAddress candidate : candidates) { + if (candidate instanceof InetSocketAddress + && ((InetSocketAddress) candidate).isUnresolved()) { LOG.debug( - "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", - shardId, - endPoint); - } - connectFuture = bootstrap.connect(resolvedAddress); - } else { - int localPort = - PortAllocator.getNextAvailablePort(shardingInfo.getShardsCount(), shardId, context); - if (localPort == -1) { - LOG.warn( - "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", - shardId, - endPoint); - connectFuture = bootstrap.connect(resolvedAddress); + "[{}] Resolver returned {} for {} but it is still unresolved, skipping it", + logPrefix, + candidate, + original); } else { - connectFuture = bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + result.add(candidate); } } + return result; + } - connectFuture.addListener( - cf -> { - if (connectFuture.isSuccess()) { - Channel channel = connectFuture.channel(); - DriverChannel driverChannel = - new DriverChannel(endPoint, channel, context.getWriteCoalescer(), currentVersion); - // If this is the first successful connection, remember the protocol version and - // cluster name for future connections. - if (isNegotiating) { - ChannelFactory.this.protocolVersion = currentVersion; - } - if (ChannelFactory.this.clusterName == null) { - ChannelFactory.this.clusterName = driverChannel.getClusterName(); - } - Map> supportedOptions = driverChannel.getOptions(); - if (ChannelFactory.this.productType == null && supportedOptions != null) { - List productTypes = supportedOptions.get("PRODUCT_TYPE"); - String productType = - productTypes != null && !productTypes.isEmpty() - ? productTypes.get(0) - : UNKNOWN_PRODUCT_TYPE; - ChannelFactory.this.productType = productType; - DriverConfig driverConfig = context.getConfig(); - if (driverConfig instanceof TypesafeDriverConfig - && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { - ((TypesafeDriverConfig) driverConfig) - .overrideDefaults( - ImmutableMap.of( - DefaultDriverOption.REQUEST_CONSISTENCY, - ConsistencyLevel.LOCAL_QUORUM.name())); - } - } - resultFuture.complete(driverChannel); - } else { - Throwable error = connectFuture.cause(); - if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { - attemptedVersions.add(currentVersion); - Optional downgraded = - context.getProtocolVersionRegistry().downgrade(currentVersion); - if (downgraded.isPresent()) { + /** + * Re-attaches the {@code original} address's host name to one of the resolved candidates it + * expanded to, whatever name that candidate carries. + * + *

The JDK and Netty-DNS resolvers already attach the queried name to the {@link InetAddress}es + * they return, so this is a no-op for them. A custom resolver, however, may build its results + * from raw address bytes, or label them with a canonical/CNAME name of its own. The channel's + * pinned endpoint is built from the candidate (see {@link PinnableEndPoint}), and it is what + * {@code DefaultSslEngineFactory} and {@code SniSslEngineFactory} derive the SSL peer host from, + * inside the channel initializer. So whatever name the candidate carries is the name TLS hostname + * verification checks the server certificate against, and the only name that may be is the one + * the user configured: with a nameless address, {@code InetSocketAddress#getHostName()} + * additionally triggers a blocking reverse-DNS lookup on the event loop and validation falls back + * to the IP or the PTR record, and with a resolver-supplied label it validates a name the + * operator never chose. Hence the queried name always wins here; before multi-address support the + * initializer kept the original endpoint and Netty resolved only the TCP destination, which had + * the same effect. + * + *

Re-attaching changes nothing else: {@code InetAddress.getByAddress(host, bytes)} performs no + * lookup, the TCP connect target is the same IP, and a resolved {@link InetSocketAddress}'s + * equality ignores host names, so pinning and the pin-equality shortcuts are unaffected. A scoped + * IPv6 candidate keeps its scope, since {@link Inet6Address} has {@code getByAddress} overloads + * that carry one. + * + *

An original that carries no name of its own is left alone (see {@link + * AddressUtils#carriesName}): a resolver is free to redirect it to a different IP, and labelling + * that IP with the literal form of the one we asked for would invent a name that resolves to + * something else. + */ + @VisibleForTesting + static SocketAddress reattachHostname(SocketAddress original, SocketAddress candidate) { + if (!(original instanceof InetSocketAddress) || !(candidate instanceof InetSocketAddress)) { + return candidate; + } + InetSocketAddress originalInet = (InetSocketAddress) original; + InetSocketAddress candidateInet = (InetSocketAddress) candidate; + InetAddress candidateIp = candidateInet.getAddress(); + if (!AddressUtils.carriesName(originalInet) + || candidateIp == null + // Nothing to change: the candidate already carries the queried name, which is the common + // case (the JDK and Netty-DNS resolvers attach it themselves). getHostString() never looks + // anything up -- for a nameless address it falls back to the IP literal. + || candidateInet.getHostString().equals(originalInet.getHostString())) { + return candidate; + } + try { + return new InetSocketAddress( + withHostName(originalInet.getHostString(), candidateIp), candidateInet.getPort()); + } catch (UnknownHostException impossible) { + // getByAddress only rejects illegal byte lengths, and these bytes come from a real + // InetAddress; keep the raw candidate rather than failing the connect over a cosmetic step. + return candidate; + } + } + + /** + * Returns a copy of {@code ip} labelled with {@code hostName}, preserving an IPv6 scope if there + * is one. + * + *

{@link InetAddress#getByAddress(String, byte[])} cannot carry a scope, and dropping one + * would change where the address actually points — a link-local address is only meaningful + * together with its zone. {@link Inet6Address#getByAddress(String, byte[], int)} carries the zone + * as its numeric id, which is what the connect itself goes on; a scope id of 0 means "unscoped" + * and is accepted, so this needs no special case for a plain IPv6 address. + * + *

The sibling overload taking a {@link NetworkInterface} is deliberately not used: it + * re-derives the numeric scope by searching that interface for an address of the same local type, + * and throws {@code UnknownHostException("no scope_id found")} when it finds none — so it can + * fail for an address that was legitimately built from an interface in the first place. All that + * is lost by going numeric is the interface name, which surfaces in {@code toString()} and + * nowhere else. + */ + private static InetAddress withHostName(String hostName, InetAddress ip) + throws UnknownHostException { + return ip instanceof Inet6Address + ? Inet6Address.getByAddress(hostName, ip.getAddress(), ((Inet6Address) ip).getScopeId()) + : InetAddress.getByAddress(hostName, ip.getAddress()); + } + + /** + * Whether {@link #shuffleAndLimit} may spread this connect across the addresses the endpoint + * expands to. + * + *

A contact point always may: its addresses may well be different nodes, so there is no + * node identity to preserve, and spreading both balances load and varies which address an attempt + * starts from. For an {@linkplain #isIdentified(Node) identified} node it depends on what the + * name denotes, which only the endpoint knows — see {@link + * PinnableEndPoint#addressesAreInterchangeable()} for the two cases and why they differ. An + * endpoint that does not implement {@link PinnableEndPoint} is treated as not interchangeable, + * which is also the conservative reading for a third-party implementation. + */ + @VisibleForTesting + static boolean spreadAcrossAddresses(EndPoint endPoint, boolean nodeIsIdentified) { + return !nodeIsIdentified + || (endPoint instanceof PinnableEndPoint + && ((PinnableEndPoint) endPoint).addressesAreInterchangeable()); + } + + /** + * Truncates the expanded address list to {@code advanced.connection.max-candidate-addresses}, + * shuffling it first when the addresses may be spread across (see {@link + * #spreadAcrossAddresses}). + * + *

The shuffle spreads load: without it, every connection would try the resolver's first + * address first and healthy connections would pile onto one IP, while the whole point of a + * multi-record name is usually to spread them. A fresh random order per connect also means + * successive attempts start at different addresses, with no per-name counter state to maintain + * and nothing depending on the order the resolver, or a sort, happened to choose. + * + *

Where the order is kept instead, it is because the addresses are not known to be + * interchangeable: an identified node whose endpoint is an unresolved name that may map to + * several hosts, which is what a configured {@code AddressTranslator} returns by default ({@code + * SubnetAddressTranslator} under {@code resolve-addresses = false}). Each pool connection is its + * own {@code connect()}, so shuffling there would land one {@code Node}'s channels on different + * hosts while routing, shard awareness and per-node metrics attribute them all to that node. + * Keeping the resolver's order means such a pool converges on one address, as it did before + * multi-address support -- {@code Bootstrap.connect()} resolved through {@code resolve()}, + * singular, i.e. the first record -- while the remaining addresses still serve as fallback. + * + *

The cap bounds what a single connect attempt can cost: every address tried is a full TCP + * connect plus init handshake -- and, with wrong credentials, a rejected login (see {@link + * #tryNextCandidate} on why an authentication failure does not stop the loop). For a shuffled + * list, a capped attempt tries a different sample of the addresses each time, so a name with more + * records than the cap still reaches all of them across successive attempts; one attempt just no + * longer walks them all. Where the order is kept, the cap is a hard limit -- a capped attempt + * keeps dialing the same prefix of the list, so records beyond it are never reached. That is + * accepted rather than worked around: it is still strictly more than the single address such an + * endpoint got before multi-address support, and rotating the window instead would give up the + * convergence the stable order exists for. + */ + @VisibleForTesting + List shuffleAndLimit( + List addresses, boolean spreadAcrossAddresses) { + List shuffled = new ArrayList<>(addresses); + if (shuffled.size() > 1 && spreadAcrossAddresses) { + Collections.shuffle(shuffled, random); + } + int cap = + Math.max( + 1, + context + .getConfig() + .getDefaultProfile() + .getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)); + if (shuffled.size() > cap) { + LOG.debug( + "[{}] Resolved {} addresses, will try at most {}" + + " (advanced.connection.max-candidate-addresses)", + logPrefix, + shuffled.size(), + cap); + return shuffled.subList(0, cap); + } + return shuffled; + } + + /** + * Iterates through the candidate addresses produced by {@link #resolveCandidates}. Tries each one + * in sequence; when an address fails, the next candidate is tried, and only when all candidates + * are exhausted is the overall {@code resultFuture} failed. + * + *

One failure is node-wide -- it dooms every remaining address rather than only the one + * that was tried: an {@link UnsupportedProtocolVersionException} against an identified + * node ({@code nodeIsIdentified}, see {@link #isIdentified(Node)}). Every address of an + * identified node is that same node, so a protocol-version rejection is a property of all of + * them, and the attempt fails immediately instead of replaying the negotiation against every + * remaining IP. This matches the pre-multi-address behaviour of a single-address connect. The + * corner it deliberately does not rescue: a heterogeneous rolling upgrade where different IPs of + * one identified node genuinely support different protocol versions. + * + *

Note what that leaves uncovered, because it is narrower than it looks: {@code isNegotiating} + * is true only while {@link #protocolVersion} is still unset, i.e. on the session's first + * connection, which is always to a contact point -- and a contact point is never identified (see + * {@link #isIdentified}). So a rejection reached by negotiation never takes this branch; + * only a forced version rejected by an already-identified node does, which is the + * single-attempt case. A negotiated rejection still walks the downgrade ladder from the top on + * every candidate, since each gets a fresh {@code attemptedVersions} list, so a name with N + * records costs N ladders (with N bounded by {@link #shuffleAndLimit}). + * + *

Every other failure -- authentication included -- advances to the next candidate: the + * addresses a name expands to may well belong to different nodes, so a rejection by the first of + * them says nothing about the rest. That also preserves the behaviour this PR would otherwise + * have removed: with {@code advanced.resolve-contact-points = true} each resolved address used to + * be a separate {@code Node}, and {@code ControlConnection} advances to the next node in its + * query plan on any error, including these. + * + *

Authentication in particular has to advance, for a reason only visible in the order of the + * handshake: {@link ProtocolInitHandler} runs {@code STARTUP -> AUTH_RESPONSE -> + * GET_CLUSTER_NAME}, so authentication completes before the cluster-name check. A stale + * DNS record pointing at a foreign cluster that wants different credentials therefore fails at + * AUTH, and treating that as terminal would write off the whole hostname -- making the + * cluster-name mismatch that would have advanced to the next address unreachable, in exactly the + * multi-record case this loop exists for. What bounds the cost of genuinely wrong credentials is + * the candidate cap ({@link #shuffleAndLimit}): one attempt pays at most {@code + * advanced.connection.max-candidate-addresses} rejected logins, with the earlier failures + * attached as suppressed exceptions. + * + *

Timeout note: addresses are tried serially, so the worst-case time before failure is + * N times a full attempt, and an attempt is a connect plus the init handshake. Each of the + * handshake's steps arms its own {@code advanced.connection.init-query-timeout} when it is sent, + * so they accumulate instead of sharing one deadline: a single address that accepts the + * connection and then stalls costs {@code connect-timeout} plus several times {@code + * init-query-timeout} before the loop moves on. This is an intentional tradeoff: failing + * immediately on the first unreachable IP would prevent fallback to healthy ones. The candidate + * cap ({@link #shuffleAndLimit}) is what bounds N. + * + *

When every candidate fails, one of their errors is propagated -- see {@link + * #surfacedFailure} for which, and why it is not simply the last -- with every other candidate's + * failure attached to it as a {@linkplain Throwable#addSuppressed(Throwable) suppressed} + * exception, so no cause is lost. + * + *

What that does not give is which address produced which failure. Every one of these + * exceptions is built from the endpoint, and a pinned copy is required to render identically to + * the unpinned original ({@link PinnableEndPoint}), so a three-record name yields three messages + * that all name the same hostname. The DEBUG line above is where the pairing lives; making the + * exceptions themselves carry it would mean either relaxing that contract or wrapping causes in a + * driver-owned type, which would in turn break the {@code instanceof} tests the callers do on + * them (see {@link #surfacedFailure}). + */ + private void tryNextCandidate( + Bootstrap baseBootstrap, + EventLoop eventLoop, + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + boolean nodeIsIdentified, + CompletableFuture resultFuture, + List candidates, + int index, + List priorErrors) { + + // Invariant: this method always (eventually) completes resultFuture. It is invoked from + // CompletionStage and Netty callbacks that swallow throwables, so a synchronous throw -- a + // custom PinnableEndPoint.pinTo() for instance -- would otherwise leave the connect attempt + // hanging forever. Double completion is harmless: completeExceptionally() on an already + // completed future is a no-op. + try { + SocketAddress candidate = candidates.get(index); + // Everything downstream of here -- the channel, its pipeline (SSL engine, authenticator) and + // the DriverChannel handed to the caller -- sees an endpoint bound to this one address + // instead of the multi-address original. See PinnableEndPoint for why that matters. + EndPoint pinnedEndPoint = pin(endPoint, candidate); + CompletableFuture perAddressFuture = new CompletableFuture<>(); + // Fresh per candidate address: connectToAddress()'s downgrade retries stay on this one + // address, so the final UnsupportedProtocolVersionException (if negotiation is what dooms + // this candidate) only reports versions actually tried against it, not earlier candidates'. + List attemptedVersions = new CopyOnWriteArrayList<>(); + connectToAddress( + baseBootstrap, + eventLoop, + pinnedEndPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + perAddressFuture, + candidate); + + perAddressFuture.whenComplete( + (channel, error) -> { + try { + boolean nodeWide = error != null && isNodeWideFailure(error, nodeIsIdentified); + if (error == null) { + if (!resultFuture.complete(channel)) { + // Same guard as completeCandidate and abandonCandidate: resultFuture is handed to + // callers as a CompletionStage and every path that can complete it early does so + // exceptionally (the blanket catches in resolveCandidates and below), so losing + // this race is possible -- and would otherwise leak a live socket and its + // pipeline for the life of the JVM, since nobody else holds this channel. + channel.forceClose(); + } + } else if (!nodeWide && index + 1 < candidates.size()) { LOG.debug( - "[{}] Failed to connect with protocol {}, retrying with {}", + "[{}] Failed to connect to {} ({}), trying next address", logPrefix, - currentVersion, - downgraded.get()); - connect( + candidate, + error.getMessage()); + priorErrors.add(error); + tryNextCandidate( + baseBootstrap, + eventLoop, + // Deliberately the original, not the pinned copy: the next candidate must be + // pinned from the unpinned endpoint. endPoint, shardingInfo, shardId, options, nodeMetricUpdater, - downgraded.get(), - true, - attemptedVersions, - resultFuture); + currentVersion, + isNegotiating, + nodeIsIdentified, + resultFuture, + candidates, + index + 1, + priorErrors); } else { - resultFuture.completeExceptionally( - UnsupportedProtocolVersionException.forNegotiation( - endPoint, attemptedVersions)); + if (index + 1 < candidates.size()) { + // Only reachable for a node-wide failure (see the javadoc). + LOG.debug( + "[{}] Not trying the remaining addresses of {}: this failure is a property of" + + " the node, not of the address ({})", + logPrefix, + endPoint, + error.getMessage()); + } + // Surface one failure, carrying the others as suppressed exceptions so they are + // not lost (they were only logged at DEBUG above). Deduplicated by identity: + // nothing stops two candidates from failing with the same Throwable instance, and + // this mutates an object we do not own -- attaching it twice would show the same + // cause twice, and would keep growing a shared instance's suppressed list on every + // connect. + List allErrors = new ArrayList<>(priorErrors); + allErrors.add(error); + Throwable surfaced = surfacedFailure(allErrors, nodeWide); + Set attached = Collections.newSetFromMap(new IdentityHashMap<>()); + attached.add(surfaced); + for (Throwable candidateError : allErrors) { + if (attached.add(candidateError)) { + surfaced.addSuppressed(candidateError); + } + } + // Note: might be completed already if the failure happened in initializer() + resultFuture.completeExceptionally(surfaced); } - } else { - // Note: might be completed already if the failure happened in initializer(), this is - // fine - resultFuture.completeExceptionally(error); + } catch (Throwable t) { + resultFuture.completeExceptionally(t); } - } - }); + }); + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } + } + + /** + * Whether {@code error} dooms every remaining address of the endpoint, making it pointless for + * {@link #tryNextCandidate} to try them. See its javadoc for the reasoning behind each case. + * + *

A {@link ClusterNameMismatchException} is deliberately absent, for an identified node as + * much as for a contact point. It says that the address just tried fronts a different cluster, + * which is a property of that record rather than of the node -- a stale DNS entry is exactly what + * it looks like -- so advancing to the next address is the whole point. {@link #surfacedFailure} + * treats it with the same caution on the way out. + * + *

An {@link AuthenticationException} is deliberately absent too, even for an identified node: + * see {@link #tryNextCandidate} on why authentication must advance, and {@link #shuffleAndLimit} + * for the cap that bounds what wrong credentials can cost. + */ + private static boolean isNodeWideFailure(Throwable error, boolean nodeIsIdentified) { + return error instanceof UnsupportedProtocolVersionException && nodeIsIdentified; + } + + /** + * Which of the candidates' failures to propagate once they are all exhausted, the rest being + * attached to it as suppressed exceptions. + * + *

Not simply the last one. Callers branch on the type of what they receive -- {@link + * com.datastax.oss.driver.internal.core.pool.ChannelPool#handleError} treats a cluster-name + * mismatch and a protocol-version rejection as fatal, an invalid keyspace as a keyspace error and + * an authentication failure as warn-and-retry; {@code ControlConnection} logs authentication + * failures differently from transport ones -- and with a multi-record name the address that + * happens to be tried last is arbitrary. Letting it win would report a firewalled IP's connect + * timeout for what is really a rejected password, and take the reconnect path where the caller + * asked for the fatal one. + * + *

So a failure the callers classify is preferred over one they do not, in the order they test + * for it, and the last non-fatal failure is only used when no candidate produced a + * classified one. Every failure is still attached, so nothing is lost either way. + * + *

The two fatal types are the exception to that: they are only preferred when every + * candidate failed that way, or when the last one is the node-wide failure that stopped the loop. + * See the comments in the body. + */ + private static Throwable surfacedFailure(List errors, boolean lastIsNodeWide) { + Throwable lastError = errors.get(errors.size() - 1); + // A node-wide failure is what ended the loop, and it is a verdict about the node rather than + // about the one address it was observed on (see tryNextCandidate). It therefore outranks + // everything below, including the unanimity rule -- which would otherwise demote it to whatever + // transport failure an earlier address happened to produce, turning the forced-down node that a + // single-address connect has always produced into a reconnect. + if (lastIsNodeWide) { + return lastError; + } + // An irreversible verdict needs evidence from every address. handleError turns these two into + // TopologyEvent.forceDown, and nothing in the driver ever reverses one -- no component fires + // FORCE_UP, and a SUGGEST_UP is explicitly refused for a FORCED_DOWN node -- so the node is out + // for the rest of the session. Meanwhile tryNextCandidate() classifies a cluster-name mismatch + // as a property of the address and advances past it, which is the point: it means this record + // is stale, not that this node belongs to another cluster. Promoting one such record over the + // other candidates' transport failures would write a healthy node off on the strength of the + // one address that was never going to work. Requiring unanimity leaves a single-address + // endpoint exactly as it was before this loop existed -- one candidate is unanimous by + // definition -- and a mixed pass simply reconnects, forcing down on the first pass that is. + boolean everyCandidateFatal = true; + for (Throwable error : errors) { + if (!isFatalToCallers(error)) { + everyCandidateFatal = false; + break; + } + } + if (everyCandidateFatal) { + return errors.get(0); + } + // An invalid keyspace, unlike those two, is a property of the cluster's schema rather than of + // the address, so one address answering settles it and no unanimity is required. It outranks an + // authentication failure because it is the rung a caller acts on -- handleError routes it to + // onKeyspaceError, which is how PoolManager fails session init fast instead of reconnecting for + // a keyspace that will never appear -- and because reaching the keyspace step at all proves the + // credentials were accepted on that address. + for (Throwable error : errors) { + if (error instanceof InvalidKeyspaceException) { + return error; + } + } + for (Throwable error : errors) { + if (error instanceof AuthenticationException) { + return error; + } + } + // The last failure -- but not a fatal one. The address tried last is arbitrary, so promoting a + // fatal failure here would force the node down on the strength of the single address that + // produced it, which is exactly what the unanimity rule above refuses to do. + for (int i = errors.size() - 1; i >= 0; i--) { + Throwable error = errors.get(i); + if (!isFatalToCallers(error)) { + return error; + } + } + // Unreachable: a list of nothing but fatal failures returned at the unanimity check above. + return lastError; + } + + /** + * Whether the callers treat {@code error} as fatal, i.e. as grounds to write the node off: {@code + * ChannelPool#handleError} turns these two, and only these two, into {@code + * TopologyEvent.forceDown}. + */ + private static boolean isFatalToCallers(Throwable error) { + return error instanceof ClusterNameMismatchException + || error instanceof UnsupportedProtocolVersionException; + } + + /** + * Performs a Netty bootstrap connect to a single, already-resolved address. Handles + * protocol-version negotiation (downgrade retries) internally, staying on the same address. Uses + * {@code perAddressFuture} so {@link #tryNextCandidate} can distinguish a per-address TCP failure + * (try the next IP) from a successful protocol handshake. + */ + private void connectToAddress( + Bootstrap baseBootstrap, + EventLoop eventLoop, + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + List attemptedVersions, + CompletableFuture perAddressFuture, + SocketAddress resolvedAddress) { + + // Invariant, as in tryNextCandidate(): every path completes perAddressFuture. The synchronous + // section can throw from Bootstrap validation; the connect listener runs inside a Netty + // callback that swallows throwables and contains the downgrade recursion, the version-registry + // lookup and the config overrides, any of which throwing would otherwise hang the attempt. + try { + // clone(eventLoop) so each attempt gets its own handler while sharing the options (including + // anything afterBootstrapInitialized() set), and is registered on the event loop the + // connect() picked -- the same one resolution ran on, so the group's chooser advances exactly + // once per logical connect (see connect()). + // + // disableResolver() because resolveCandidates() has already done the one resolution pass this + // connect gets, and `resolvedAddress` is one of its results. Bootstrap.clone() otherwise + // carries the resolver over and Netty resolves again -- through resolve(), *singular*. That + // is inert for the default resolver, which short-circuits on isResolved(), but a resolver + // that reports resolved addresses as unresolved in order to redirect them -- which + // resolveCandidates() deliberately supports -- would remap every candidate onto its first + // answer: the remaining candidates would never actually be tried, and the endpoint pinned + // onto the channel would name an address the channel is not connected to (which is what the + // SSL engine's peer host and DefaultTopologyMonitor#savePort are derived from). Every other + // exit from resolveCandidates() yields an address Netty would itself have passed through + // untouched -- no group, !isSupported, or isResolved -- so nothing else changes. + Bootstrap bootstrap = + baseBootstrap + .clone(eventLoop) + .disableResolver() + .handler( + initializer( + endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); + + ChannelFuture connectFuture; + if (shardId == null || shardingInfo == null) { + if (shardId != null) { + LOG.debug( + "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", + shardId, + endPoint); + } + connectFuture = bootstrap.connect(resolvedAddress); + } else { + int localPort = + PortAllocator.getNextAvailablePort(shardingInfo.getShardsCount(), shardId, context); + if (localPort == -1) { + LOG.warn( + "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", + shardId, + endPoint); + connectFuture = bootstrap.connect(resolvedAddress); + } else { + connectFuture = bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + } + } + + connectFuture.addListener( + cf -> { + try { + if (connectFuture.isSuccess()) { + Channel channel = connectFuture.channel(); + DriverChannel driverChannel = + new DriverChannel( + endPoint, channel, context.getWriteCoalescer(), currentVersion); + // If this is the first successful connection, remember the protocol version and + // cluster name for future connections. + if (isNegotiating) { + ChannelFactory.this.protocolVersion = currentVersion; + } + if (ChannelFactory.this.clusterName == null) { + ChannelFactory.this.clusterName = driverChannel.getClusterName(); + } + Map> supportedOptions = driverChannel.getOptions(); + if (ChannelFactory.this.productType == null && supportedOptions != null) { + List productTypes = supportedOptions.get("PRODUCT_TYPE"); + String productType = + productTypes != null && !productTypes.isEmpty() + ? productTypes.get(0) + : UNKNOWN_PRODUCT_TYPE; + ChannelFactory.this.productType = productType; + DriverConfig driverConfig = context.getConfig(); + if (driverConfig instanceof TypesafeDriverConfig + && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { + ((TypesafeDriverConfig) driverConfig) + .overrideDefaults( + ImmutableMap.of( + DefaultDriverOption.REQUEST_CONSISTENCY, + ConsistencyLevel.LOCAL_QUORUM.name())); + } + } + perAddressFuture.complete(driverChannel); + } else { + Throwable error = connectFuture.cause(); + if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { + attemptedVersions.add(currentVersion); + Optional downgraded = + context.getProtocolVersionRegistry().downgrade(currentVersion); + if (downgraded.isPresent()) { + LOG.debug( + "[{}] Failed to connect with protocol {}, retrying with {}", + logPrefix, + currentVersion, + downgraded.get()); + // Stay on the same address for protocol-version downgrade retries. + connectToAddress( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + downgraded.get(), + true, + attemptedVersions, + perAddressFuture, + resolvedAddress); + } else { + perAddressFuture.completeExceptionally( + UnsupportedProtocolVersionException.forNegotiation( + endPoint, attemptedVersions)); + } + } else { + // Note: might be completed already if the failure happened in initializer(), this + // is fine + perAddressFuture.completeExceptionally(error); + } + } + } catch (Throwable t) { + // Close the channel we opened before giving up on it. Nothing else holds it once this + // listener returns -- the DriverChannel wrapper is out of scope, and no candidate was + // completed with it -- so the socket and its pipeline would stay open for the life of + // the JVM. Only when the future is ours to fail, though: a candidate that completed + // successfully is the caller's channel, not ours to close. + if ((perAddressFuture.completeExceptionally(t) + || perAddressFuture.isCompletedExceptionally()) + && connectFuture.isSuccess()) { + connectFuture.channel().close(); + } + } + }); + } catch (Throwable t) { + perAddressFuture.completeExceptionally(t); + } + } + + /** + * Binds {@code endPoint} to the address a connection is being opened to, when the implementation + * supports it. + * + *

Third-party {@link EndPoint}s that do not implement {@link PinnableEndPoint} are returned + * unchanged, so they keep behaving exactly as they did before multi-address support: the channel + * carries the endpoint it was given. + * + *

So is an endpoint whose candidate came back unresolved. {@link + * PinnableEndPoint#pinTo(SocketAddress)} is documented to take an address that is already + * resolved, and {@link #resolveCandidates} has three paths that hand the original address + * straight through -- the user disabled the resolver, the resolver does not support the address, + * or it reports it as already resolved. For an endpoint that hands out a hostname, pinning there + * would freeze it on a name that still re-expands on every connect: no address stability gained, + * and whatever the endpoint does instead of consulting its own source once pinned is lost. + */ + private static EndPoint pin(EndPoint endPoint, SocketAddress resolvedAddress) { + if (resolvedAddress instanceof InetSocketAddress + && ((InetSocketAddress) resolvedAddress).isUnresolved()) { + return endPoint; + } + return endPoint instanceof PinnableEndPoint + ? ((PinnableEndPoint) endPoint).pinTo(resolvedAddress) + : endPoint; } @VisibleForTesting @@ -463,7 +1349,11 @@ protected void initChannel(Channel channel) { context.getNettyOptions().afterChannelInitialized(channel); } catch (Throwable t) { // If the init handler throws an exception, Netty swallows it and closes the channel. We - // want to propagate it instead, so fail the outer future (the result of connect()). + // want to propagate it instead, so fail this candidate's future. Note that is the + // per-address one, not the result of connect(): a pipeline failure that is not specific to + // the address (a bad truststore, say) therefore advances to the next candidate and is + // retried against each of them, which tryNextCandidate() documents as the deliberate + // trade-off for not being able to tell the two apart. resultFuture.completeExceptionally(t); throw t; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java index 5b4ff4dcec8..b319186910d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java @@ -66,7 +66,21 @@ public interface NettyOptions { /** * A hook invoked each time the driver creates a client bootstrap in order to open a channel. This - * is a good place to configure any custom option on the bootstrap. + * is a good place to configure any custom option, attribute, or {@link + * Bootstrap#resolver(io.netty.resolver.AddressResolverGroup)} on the bootstrap. + * + *

The hook runs once per logical connection to a node. When a hostname expands to several IP + * addresses, the same bootstrap is shared by every per-address attempt (each attempt uses a + * {@link Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it); likewise, protocol-version + * downgrade retries reuse it. Before multi-address support the hook ran once per attempt, + * including once per downgrade retry. + * + *

The bootstrap does not carry the driver's channel handler yet, and a handler + * installed by this hook is not honoured: the driver sets its own handler on each + * per-attempt copy afterwards (and logs a one-time warning if it overwrites one). To customize + * the pipeline, use {@link #afterChannelInitialized(Channel)} instead. (Before multi-address + * support the hook ran after the driver's handler was installed, so replacing it was technically + * possible; that was never a supported extension point.) */ void afterBootstrapInitialized(Bootstrap bootstrap); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java index 9da3a2a8aa2..51cddf0b728 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java @@ -45,6 +45,7 @@ import com.datastax.oss.driver.internal.core.util.concurrent.Reconnection; import com.datastax.oss.driver.internal.core.util.concurrent.RunOrSchedule; import com.datastax.oss.driver.internal.core.util.concurrent.UncaughtExceptions; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.Message; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -752,24 +753,52 @@ private void forceClose() { } } - private boolean isAuthFailure(Throwable error) { - if (error instanceof AllNodesFailedException) { - Collection> errors = - ((AllNodesFailedException) error).getAllErrors().values(); - if (errors.isEmpty()) { - return false; - } - for (List nodeErrors : errors) { - for (Throwable nodeError : nodeErrors) { - if (!(nodeError instanceof AuthenticationException)) { - return false; - } + /** + * Whether every contact point failed for the one reason worth telling the operator to go and fix + * their configuration over: bad credentials, everywhere. + * + *

Each entry is tested with {@link #isAuthOnly} rather than a bare {@code instanceof}, because + * one entry no longer means one address. {@code ChannelFactory} expands a contact-point hostname + * to every address it resolves to and reports a single failure for the name, with the other + * addresses' failures attached as suppressed exceptions. Looking only at the top-level throwable + * would call a name whose records failed {@code [refused, refused, auth]} an authentication + * failure, and claim in the log that authentication is what is wrong with the deployment when two + * thirds of it is unreachable. + */ + @VisibleForTesting + static boolean isAuthFailure(Throwable error) { + if (!(error instanceof AllNodesFailedException)) { + // Anything else carries no per-node breakdown to inspect, so there is nothing here that says + // every contact point rejected the credentials. + return false; + } + Collection> errors = ((AllNodesFailedException) error).getAllErrors().values(); + if (errors.isEmpty()) { + return false; + } + for (List nodeErrors : errors) { + for (Throwable nodeError : nodeErrors) { + if (!isAuthOnly(nodeError)) { + return false; } } } return true; } + /** Whether {@code error} and every failure attached to it are authentication failures. */ + private static boolean isAuthOnly(Throwable error) { + if (!(error instanceof AuthenticationException)) { + return false; + } + for (Throwable suppressed : error.getSuppressed()) { + if (!(suppressed instanceof AuthenticationException)) { + return false; + } + } + return true; + } + /** * Immutable snapshot of the control node state. Reads from any thread see a consistent pair of * (current, pending) via a single volatile read of the enclosing reference. diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 4a13886c9d9..72a1a7b7a97 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -541,6 +541,23 @@ datastax-java-driver { # Overridable in a profile: no max-orphan-requests = 256 + # The maximum number of addresses a single connection attempt will try, when the endpoint it + # connects to is a DNS name that resolves to several addresses. + # + # The addresses are tried in random order, and each one tried is a full TCP connect plus + # protocol handshake -- with wrong credentials, that includes a rejected login. This cap bounds + # what one attempt can cost in time and in login attempts. Addresses beyond the cap are not + # lost: the order is shuffled again on every attempt, so successive attempts (e.g. reconnection + # rounds) sample different subsets. + # + # Setting this to 1 restores pre-multi-address behavior: one address tried per attempt. + # + # Required: yes + # Modifiable at runtime: yes, the new value will be used for connections created after the + # change. + # Overridable in a profile: no + max-candidate-addresses = 5 + # Whether to log non-fatal errors when the driver tries to open a new connection. # # This error as recoverable, as the driver will try to reconnect according to the reconnection diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java new file mode 100644 index 00000000000..dbe60ae4432 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.context.NettyOptions; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelInboundHandlerAdapter; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies the {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} contract: the hook runs on + * a handler-less bootstrap, and a handler it installs is replaced by the driver's own. + */ +public class ChannelFactoryBootstrapHookTest extends ChannelFactoryTestBase { + + @Test + public void should_replace_handler_installed_by_bootstrap_hook() { + // Given – a hook that (incorrectly) installs its own channel handler. The driver sets its own + // handler on each per-attempt copy afterwards, logging a one-time warning; if the dummy + // handler below survived instead, the protocol handshake would never happen and this connect + // would fail on the init timeout. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.handler(new ChannelInboundHandlerAdapter()); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java new file mode 100644 index 00000000000..d62d9bc9d58 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -0,0 +1,816 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metadata.SniEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import com.datastax.oss.protocol.internal.Frame; +import com.datastax.oss.protocol.internal.request.Options; +import com.datastax.oss.protocol.internal.request.Startup; +import com.datastax.oss.protocol.internal.response.Authenticate; +import com.datastax.oss.protocol.internal.response.Ready; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.channel.local.LocalAddress; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.NetworkInterface; +import java.net.SocketAddress; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.Test; + +/** + * Verifies how {@link ChannelFactory#connect} treats the several addresses a name expands to: they + * are tried in sequence in a shuffled order, at most {@code + * advanced.connection.max-candidate-addresses} of them, and failures are aggregated rather than + * dropped. + * + *

The expansion itself is exercised in {@link ChannelFactoryNettyResolverTest}; here the + * resolver is only the mechanism for producing more than one address from a single endpoint. + */ +public class ChannelFactoryMultiAddressTest extends ChannelFactoryTestBase { + + // Local addresses that no server is bound to: connecting to them fails immediately. + private static final SocketAddress UNREACHABLE_1 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-1"); + private static final SocketAddress UNREACHABLE_2 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-2"); + private static final SocketAddress UNREACHABLE_3 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-3"); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + @Test + public void should_fail_with_suppressed_causes_when_all_addresses_are_unreachable() { + // Given – a name that expands to two dead addresses. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – the future fails, and the earlier address's failure is preserved as a suppressed + // exception on the last one's error rather than being silently dropped. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e.getSuppressed()) + .as("earlier address failures should be attached as suppressed exceptions") + .isNotEmpty()); + } + + @Test + public void should_attach_each_earlier_failure_at_most_once() { + // Given – three dead addresses, so there are earlier failures to carry. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – attaching to the error being reported mutates an object the driver does not own, so + // every cause has to appear at most once and the error must never suppress itself. Nothing + // stops two candidates from failing with the same instance -- a pipeline handler that throws a + // stackless singleton, say -- and such an instance would otherwise grow a suppressed entry on + // every connect for as long as the JVM lives. + assertThatStage(channelFuture) + .isFailed( + e -> { + List suppressed = Arrays.asList(e.getSuppressed()); + assertThat(suppressed).isNotEmpty(); + for (int i = 0; i < suppressed.size(); i++) { + assertThat(suppressed.get(i)).isNotSameAs(e); + for (int j = i + 1; j < suppressed.size(); j++) { + assertThat(suppressed.get(i)).isNotSameAs(suppressed.get(j)); + } + } + }); + } + + @Test + public void should_try_next_address_when_authentication_fails_on_a_contact_point() { + // Given – a name expanding to two addresses, both the same live server, which asks for + // authentication the driver has no provider for. The endpoint is a bare contact point, so the + // driver does not yet know which node -- or even which cluster -- any of these addresses is. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // Then – the loop advances. Authentication completes before the cluster-name check + // (ProtocolInitHandler runs STARTUP -> AUTH_RESPONSE -> GET_CLUSTER_NAME), so a stale record + // pointing at a foreign cluster that wants different credentials fails here rather than at the + // cluster-name mismatch that would have advanced. Writing off the whole name on this error + // would therefore make that rule unreachable in exactly the multi-record case this loop is for. + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message) + .as("the second candidate should have been attempted") + .isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // And – once both are exhausted the failure is still an AuthenticationException, with the first + // address's copy attached rather than dropped. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the first candidate's failure should be attached as suppressed") + .hasSize(1); + }); + } + + @Test + public void should_surface_the_authentication_failure_when_another_address_fails_on_transport() { + // Given – a name expanding to the live server (which asks for authentication the driver has no + // provider for) and a dead address. The shuffled order does not matter: whichever is tried + // first, the pass ends with one authentication failure and one transport failure. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, UNREACHABLE_1))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // Then – even when the transport failure is the *last* error, propagating it would report a + // connect failure for what is really a rejected password: callers branch on the type of what + // they receive (ChannelPool#handleError, ControlConnection's auth-specific warning and its + // errors.connection.auth metric), and with a shuffled multi-record name which address happens + // to be tried last is arbitrary. The classified failure wins, and the transport one is still + // attached. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e) + .as("an authentication failure must not be demoted by a later transport failure") + .isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the transport failure should still be attached") + .hasSize(1); + assertThat(e.getSuppressed()[0]).isNotInstanceOf(AuthenticationException.class); + }); + } + + @Test + public void should_not_surface_a_cluster_name_mismatch_that_only_one_address_reported() { + // Given – a factory that already knows the cluster name (from a first connection), then a name + // expanding to the live server -- which now answers with a *different* cluster name -- and a + // dead address. Whichever order the shuffle picks, the pass ends with one cluster-name mismatch + // and one transport failure. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + CompletionStage firstChannel = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + assertThatStage(firstChannel).isSuccess(); + installResolver( + new TestAddressResolverGroup(Arrays.asList(SERVER_ADDRESS.resolve(), UNREACHABLE_1))); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // The dead address sends nothing, so this drives the live candidate whichever position it got. + // The protocol version and the product type are known by now, hence no OPTIONS request. + writeInboundFrame(readOutboundFrame(), new Ready()); + writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("wrongClusterName")); + + // Then – the mismatch must not be the failure that surfaces, not even as the last error of the + // pass. ChannelPool#handleError turns it into TopologyEvent.forceDown and nothing in the driver + // ever reverses one, while one address of a multi-record name fronting another cluster is a + // stale record rather than a verdict about the node. It is still attached, so nothing is lost. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e) + .as("a mismatch from a single address must not be promoted over the others") + .isNotInstanceOf(ClusterNameMismatchException.class); + assertThat( + Arrays.stream(e.getSuppressed()) + .anyMatch(s -> s instanceof ClusterNameMismatchException)) + .as("the mismatch should still be attached as a suppressed exception") + .isTrue(); + }); + } + + @Test + public void should_try_next_address_when_authentication_fails_on_an_identified_node() { + // Given – the same server and the same two addresses, but a node the driver has already + // identified (nodeIsIdentified = true, i.e. its host id was read from system.local/peers). + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + failAuthenticationOnNextCandidate(); + + // Then – the loop advances, exactly as it does for a contact point: no single address's + // failure writes off the endpoint, and the candidate cap -- not a node-wide classification -- + // is what bounds the cost of genuinely wrong credentials. + failAuthenticationOnNextCandidate(); + + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the first candidate's failure should be attached as suppressed") + .hasSize(1); + }); + } + + /** Drives one candidate's handshake as far as the server's authentication challenge. */ + private void failAuthenticationOnNextCandidate() { + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + } + + @Test + public void should_stop_after_the_configured_number_of_addresses() { + // Given – a name expanding to three addresses, but a cap of two. Every address tried is a full + // connect plus handshake -- and, with wrong credentials, a rejected login -- and the + // reconnection fallback re-appends the contact points to every round, so an unbounded walk + // would repeat per contact point, per round, for as long as the session lives. The cap is what + // bounds that. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – only two addresses were dialed. The count of suppressed entries is not what to assert + // on: a single dead candidate can contribute two entangled failures (the transport refusal, + // plus an init-write failure that PromiseCombiner attaches to it as suppressed). The set of + // addresses named anywhere in the aggregate is what reflects the dials. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(mentionedUnreachableAddresses(e)) + .as("a cap of 2 means exactly two addresses dialed") + .hasSize(2)); + } + + private static final Pattern UNREACHABLE_NAME = Pattern.compile("unreachable-\\d"); + + /** The distinct dead-address names mentioned anywhere in {@code error}'s suppressed tree. */ + private static Set mentionedUnreachableAddresses(Throwable error) { + Set names = new HashSet<>(); + Deque toVisit = new ArrayDeque<>(); + toVisit.push(error); + while (!toVisit.isEmpty()) { + Throwable current = toVisit.pop(); + String message = current.getMessage(); + if (message != null) { + Matcher matcher = UNREACHABLE_NAME.matcher(message); + while (matcher.find()) { + names.add(matcher.group()); + } + } + for (Throwable suppressed : current.getSuppressed()) { + toVisit.push(suppressed); + } + } + return names; + } + + @Test + public void should_shuffle_candidates_without_losing_any() { + // The order is random per connect -- that is what spreads load across a name's records and + // varies the starting address between successive attempts -- but every address must survive + // the shuffle, since the loop's fallback walks this list. + ChannelFactory factory = newChannelFactory(); + + assertThat( + factory.shuffleAndLimit( + Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3), true)) + .containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + } + + @Test + public void should_order_candidates_by_the_injected_random_source() { + // The injection point for ordering-sensitive tests: a seeded Random produces the same + // permutation on two factories, so a scenario that needs a particular order picks a seed + // instead of depending on a sort the production code no longer performs. + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + ChannelFactory one = newChannelFactory(); + ChannelFactory other = newChannelFactory(); + one.random = new Random(42); + other.random = new Random(42); + + assertThat(one.shuffleAndLimit(addresses, true)) + .containsExactlyElementsOf(other.shuffleAndLimit(addresses, true)); + } + + @Test + public void should_leave_a_single_address_alone() { + ChannelFactory factory = newChannelFactory(); + + assertThat(factory.shuffleAndLimit(Collections.singletonList(UNREACHABLE_1), true)) + .containsExactly(UNREACHABLE_1); + } + + @Test + public void should_truncate_the_shuffled_list_to_the_cap() { + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + ChannelFactory factory = newChannelFactory(); + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + + List capped = factory.shuffleAndLimit(addresses, true); + + assertThat(capped).hasSize(2); + assertThat(addresses).containsAll(capped); + } + + @Test + public void should_clamp_the_cap_to_at_least_one_address() { + // Zero or a negative value cannot mean "dial nothing" -- the attempt would fail without ever + // trying an address. It degrades to the pre-multi-address behavior of one address per attempt. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(0); + ChannelFactory factory = newChannelFactory(); + + assertThat(factory.shuffleAndLimit(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2), true)) + .hasSize(1); + } + + @Test + public void should_not_shuffle_when_the_addresses_are_not_interchangeable() { + // A name that may denote different hosts -- what an AddressTranslator can hand back, and + // SubnetAddressTranslator does by default -- must keep the resolver's order: a random one would + // scatter a single Node's pool across hosts that routing, shard awareness and per-node metrics + // all attribute to that one node. Keeping the order makes such a pool converge on one address, + // as it did before multi-address support, while the rest of the list still serves as fallback. + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + ChannelFactory factory = newChannelFactory(); + // A seed that does permute this list, so the assertion below fails if the shuffle still runs. + factory.random = new Random(42); + assertThat(factory.shuffleAndLimit(addresses, true)).isNotEqualTo(addresses); + + assertThat(factory.shuffleAndLimit(addresses, false)).containsExactlyElementsOf(addresses); + } + + @Test + public void should_still_cap_the_candidates_when_the_order_is_kept() { + // The cap is what bounds the cost of one attempt, and that applies whether or not the addresses + // were shuffled. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + ChannelFactory factory = newChannelFactory(); + + assertThat( + factory.shuffleAndLimit( + Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3), false)) + .containsExactly(UNREACHABLE_1, UNREACHABLE_2); + } + + // ---- spreadAcrossAddresses() ---------------------------------------------- + + /** The endpoints below only have to exist; nothing in this section connects to them. */ + private static final InetSocketAddress SOME_ADDRESS = + InetSocketAddress.createUnresolved("node.example.com", 9042); + + @Test + public void should_spread_across_the_addresses_of_a_contact_point() { + // Nothing is known about a contact point's addresses -- they may be different nodes -- so there + // is no node identity to preserve and every shape is spread. + assertThat(ChannelFactory.spreadAcrossAddresses(new DefaultEndPoint(SOME_ADDRESS), false)) + .isTrue(); + assertThat( + ChannelFactory.spreadAcrossAddresses( + new SniEndPoint(SOME_ADDRESS, "server-name"), false)) + .isTrue(); + } + + @Test + public void should_spread_across_the_addresses_of_an_identified_node_behind_a_proxy() { + // An SNI proxy routes by server name, so every one of its A-records reaches this same node. + // Spreading is what SniEndPoint#resolve() itself did, by rotating through the sorted records, + // before resolution moved to the connection layer. + assertThat( + ChannelFactory.spreadAcrossAddresses( + new SniEndPoint(SOME_ADDRESS, "server-name"), true)) + .isTrue(); + } + + @Test + public void should_keep_the_order_for_an_identified_node_on_a_plain_endpoint() { + // The case the interchangeability flag exists to exclude: a DefaultEndPoint holding a name an + // AddressTranslator supplied carries no guarantee that its addresses are one node. + assertThat(ChannelFactory.spreadAcrossAddresses(new DefaultEndPoint(SOME_ADDRESS), true)) + .isFalse(); + } + + @Test + public void should_keep_the_order_for_an_identified_node_on_a_third_party_endpoint() { + // An EndPoint that does not implement PinnableEndPoint cannot say, and the conservative reading + // is the one that preserves node identity. + EndPoint thirdParty = mock(EndPoint.class); + when(thirdParty.resolve()).thenReturn(SOME_ADDRESS); + + assertThat(ChannelFactory.spreadAcrossAddresses(thirdParty, true)).isFalse(); + } + + // ---- reattachHostname() --------------------------------------------------- + + @Test + public void should_reattach_queried_hostname_to_nameless_resolved_address() throws Exception { + // A custom resolver may build its results from raw address bytes; the queried name must be + // re-attached so TLS hostname validation checks the configured name (not the IP or a PTR + // record) and reading the host name never triggers a reverse lookup on the event loop. + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9999); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.isUnresolved()).isFalse(); + // getHostString() never looks anything up; getHostName() reverse-resolves a *nameless* + // address, so it returning the queried name proves the name is embedded, not looked up. + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getHostName()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + // The candidate's port wins over the original's: a resolver may remap ports too. + assertThat(result.getPort()).isEqualTo(9999); + // Equality is unchanged (a resolved InetSocketAddress compares IP bytes + port only), so + // pinning and the pin-equality shortcuts behave exactly as with the raw candidate. + assertThat(result).isEqualTo(candidate); + } + + @Test + public void should_override_resolver_provided_hostname_with_queried_name() throws Exception { + // A resolver may label its results with a canonical/CNAME name of its own. That name would end + // up on the pinned endpoint and hence be the one TLS hostname verification checks the server + // certificate against, so the name the user configured has to win over it. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("cname.example.fake", new byte[] {10, 0, 0, 1}), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_pass_candidate_through_when_it_already_carries_the_queried_name() + throws Exception { + // The common case: the JDK and Netty-DNS resolvers attach the queried name themselves, so + // there is nothing to rebuild. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("test.cluster.fake", new byte[] {10, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(HOSTNAME, candidate)).isSameAs(candidate); + } + + @Test + public void should_pass_non_inet_candidate_through() { + // The local-transport addresses these unit tests connect over must never be touched. + assertThat(ChannelFactory.reattachHostname(HOSTNAME, UNREACHABLE_1)).isSameAs(UNREACHABLE_1); + } + + @Test + public void should_pass_candidate_through_when_original_carries_no_name() throws Exception { + // An original written as an IP literal has no name to carry over, and inventing one from the + // literal would be worse than leaving the candidate alone: a resolver is free to redirect it to + // a different IP, which would then be labelled with the literal form of a *different* address. + InetSocketAddress original = new InetSocketAddress("127.0.0.1", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + assertThat(AddressUtils.carriesName(original)).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("10.0.0.1", 9042))) + .isFalse(); + } + + @Test + public void should_reattach_name_of_a_resolved_original() throws Exception { + // A resolved original still reaches the resolver -- whether an address needs resolving is the + // resolver's call, and a custom one may redirect it. Its name is the one the operator + // configured, so it must survive onto whatever the resolver substitutes, exactly as it did when + // Netty resolved the TCP destination and the channel kept the original endpoint for TLS. + InetSocketAddress original = new InetSocketAddress("localhost", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + assertThat(AddressUtils.carriesName(original)).isTrue(); + assertThat(result.getHostString()).isEqualTo("localhost"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + } + + @Test + public void should_reattach_hostname_to_nameless_ipv6_address() throws Exception { + byte[] loopback = new byte[16]; + loopback[15] = 1; // ::1 + InetSocketAddress candidate = new InetSocketAddress(InetAddress.getByAddress(loopback), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isEqualTo(candidate.getAddress()); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_keep_the_scope_when_reattaching_to_a_scoped_ipv6_address() throws Exception { + // A link-local address only points anywhere together with its zone, so the queried name has to + // be re-attached without dropping the scope. InetAddress.getByAddress(host, bytes) cannot carry + // one, but Inet6Address.getByAddress(host, bytes, scopeId) can. + byte[] linkLocal = new byte[16]; + linkLocal[0] = (byte) 0xfe; + linkLocal[1] = (byte) 0x80; + linkLocal[15] = 1; + InetSocketAddress candidate = + new InetSocketAddress(Inet6Address.getByAddress(null, linkLocal, 3), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isInstanceOf(Inet6Address.class); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(3); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_keep_the_zone_of_an_interface_scoped_ipv6_address() throws Exception { + // An address built from a NetworkInterface rather than from an index must keep pointing into + // the + // same zone. The numeric scope the JDK derived at construction is what the connect goes on, so + // carrying that over is enough; only the interface name, a toString() detail, is not. + Inet6Address linkLocal = firstInterfaceScopedIpv6Address(); + assumeThat(linkLocal).as("no interface-scoped IPv6 address on this host").isNotNull(); + InetSocketAddress candidate = new InetSocketAddress(linkLocal, 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(linkLocal.getScopeId()); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal.getAddress()); + } + + /** An interface-scoped IPv6 address of this host, or null if it has none. */ + private static Inet6Address firstInterfaceScopedIpv6Address() throws Exception { + for (NetworkInterface nif : Collections.list(NetworkInterface.getNetworkInterfaces())) { + for (InetAddress address : Collections.list(nif.getInetAddresses())) { + if (address instanceof Inet6Address + && ((Inet6Address) address).getScopedInterface() != null) { + return (Inet6Address) address; + } + } + } + return null; + } + + @Test + public void should_fail_future_when_endpoint_resolve_throws() { + // ChannelFactory calls EndPoint.resolve() directly on the caller thread, so a third-party + // implementation that throws must surface as a failed future rather than an escaping exception. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + IllegalStateException failure = new IllegalStateException("resolve() blew up"); + + CompletionStage channelFuture = + factory.connect( + new ThrowingEndPoint(failure), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + @Test + public void should_fail_future_when_endpoint_resolve_returns_null() { + // EndPoint.resolve() is contractually non-null, but a broken third-party implementation must + // fail fast rather than NPE later inside an event-loop task, which would leave the future + // hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new NullResolvingEndPoint(), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("returned null")); + } + + @Test + public void should_fail_future_when_event_loop_group_is_rejecting_tasks() + throws InterruptedException { + // Resolution is dispatched to an I/O event loop; if the group is already shutting down, that + // dispatch is rejected synchronously. The rejection must fail the future rather than escape to + // the caller (connect() never used to throw) or leave the future hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + clientGroup.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS).sync(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).isInstanceOf(RejectedExecutionException.class)); + } + + /** An endpoint whose {@link EndPoint#resolve()} throws, standing in for a broken third party. */ + private static class ThrowingEndPoint implements EndPoint { + + private final RuntimeException failure; + + ThrowingEndPoint(RuntimeException failure) { + this.failure = failure; + } + + @NonNull + @Override + public SocketAddress resolve() { + throw failure; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } + + /** A broken third-party endpoint that violates {@code resolve()}'s non-null contract. */ + private static class NullResolvingEndPoint implements EndPoint { + + @NonNull + @Override + @SuppressWarnings("NullAway") // deliberately broken, that is the point of the test + public SocketAddress resolve() { + return null; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java new file mode 100644 index 00000000000..540962b4aeb --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.DefaultEventLoopGroup; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +/** + * Verifies that {@link ChannelFactory} expands unresolved candidate addresses through Netty's + * configured {@link AddressResolverGroup}, rather than doing its own JVM DNS lookup. + * + *

This is what keeps a custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)} + * effective: before multi-address support, an unresolved address was handed straight to {@code + * Bootstrap.connect()} and Netty's resolver expanded it, so resolving anywhere else would silently + * bypass the user's configuration. + */ +public class ChannelFactoryNettyResolverTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryNettyResolverTest.class.getSimpleName() + "-unreachable"); + + /** The hostname the endpoint reports, and that only the custom resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + /** What a resolver must never hand back from {@code resolveAll}, but might. */ + private static final InetSocketAddress STILL_UNRESOLVED = + InetSocketAddress.createUnresolved("still.unresolved.fake", 9042); + + @Test + public void should_expand_unresolved_address_through_the_custom_netty_resolver() { + // Given – a resolver that maps the hostname to an unreachable address followed by the running + // local server, mimicking a DNS round-robin entry whose first record is dead. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint itself performs no resolution at all; it just yields the hostname. + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + // The handshake only happens once we fall back to the reachable second address. + completeSimpleChannelInit(); + + // Then – the custom resolver was consulted for the hostname, and *all* the addresses it + // returned + // were tried, so the connection survived the dead first record. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the custom Netty resolver must be the one expanding the hostname") + .containsExactly(HOSTNAME); + } + + @Test + public void should_fail_when_the_custom_resolver_cannot_resolve_the_only_candidate() { + // Given – a resolver that fails every lookup. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = new TestAddressResolverGroup(null); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – no candidate survived resolution, so the connect fails with the resolver's own cause + // rather than, say, an empty-candidate-list error. + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).hasMessageContaining("mock resolver failure")); + } + + @Test + public void should_fail_with_a_diagnosable_error_when_every_expanded_address_is_unresolved() { + // Given – a resolver that "expands" the hostname to another unresolved address. A redirecting + // resolver can do this by rewriting the host without resolving it, and nothing downstream will + // resolve it either: connectToAddress() uses a bootstrap clone with disableResolver(), so Netty + // would raise UnresolvedAddressException from inside doConnect, naming neither the address nor + // the reason nothing resolved it -- for every connect of the whole session. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(new TestAddressResolverGroup(Collections.singletonList(STILL_UNRESOLVED))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – the failure says which endpoint and which resolver produced it, as the pass-through + // paths already did (see ChannelFactory#unusableWithoutResolution). + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(IllegalStateException.class); + assertThat(e.getMessage()).contains("test.cluster.fake"); + assertThat(e.getMessage()).contains("TestAddressResolverGroup"); + assertThat(e.getMessage()).contains("unresolved"); + }); + } + + @Test + public void should_drop_an_unresolved_expanded_address_before_applying_the_cap() { + // Given – the same resolver answering with one unusable address and one live one, and a cap of + // a + // single candidate. An address that cannot be connected to must not consume a slot in that cap: + // dropped after the truncation instead, it would leave this connect with nothing to dial. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(1); + installResolver( + new TestAddressResolverGroup(Arrays.asList(STILL_UNRESOLVED, SERVER_ADDRESS.resolve()))); + ChannelFactory factory = newChannelFactory(); + // Keeping the resolver's order is what makes this an assertion about the cap rather than about + // luck: the unusable address is the one the truncation would otherwise have kept. + factory.random = new KeepResolverOrder(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_not_resolve_at_all_when_the_user_disabled_the_resolver() { + // Given – Bootstrap.disableResolver() means config().resolver() is null. ChannelFactory must + // treat that as "pass the candidates through" instead of dereferencing the missing group. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(resolverGroup).disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint yields an already-usable address. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – connection succeeds and the resolver was never even instantiated, let alone consulted. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.resolverRequested).isFalse(); + assertThat(resolverGroup.queried).isEmpty(); + } + + @Test + public void should_pass_already_resolved_address_through_untouched() { + // Given – an endpoint whose address is already resolved, which is the common case: metadata + // nodes hold resolved addresses from the peers rows, so this is every pool refill and every + // reconnect. A resolver with the usual semantics reports it as resolved and there is nothing + // to expand. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – no lookup was performed: had one been, it would have redirected us to UNREACHABLE and + // the connection would have failed. The decision was the resolver's own, though -- see + // should_let_the_resolver_redirect_an_already_resolved_address. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried).isEmpty(); + assertThat(resolverGroup.resolverRequested) + .as("whether an address needs resolving must be the resolver's decision") + .isTrue(); + } + + @Test + public void should_let_the_resolver_redirect_an_already_resolved_address() { + // Given – a resolver that reports even an address carrying an IP as still needing resolution, + // and redirects it. Netty consulted the resolver for every connect, resolved address or not + // (Bootstrap#doResolveAndConnect0 calls isSupported()/isResolved() on it rather than testing + // the address itself), so short-circuiting on InetSocketAddress#isUnresolved() here would take + // that away for every connect to an already-resolved node -- which is nearly all of them. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup( + Collections.singletonList(SERVER_ADDRESS.resolve()), + /* claimNothingIsResolved = */ true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint holds a resolved address that nothing is listening on. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(resolved), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – the connect landed on the address the resolver substituted, which it could only do by + // having been asked about an address that already carried an IP. Exactly one lookup: the + // per-attempt bootstrap has the resolver disabled, so the substitute is connected to as-is + // rather than being handed back to the resolver (see the next test for why that matters). + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the resolver must get a say on an address that already carries an IP") + .containsExactly(resolved); + } + + @Test + public void should_try_every_candidate_when_the_resolver_redirects() { + // Given – the same redirecting resolver as above, but answering with more than one address: + // a dead one first, then the running local server. + // + // The per-attempt bootstrap must not re-resolve. Bootstrap.clone() carries the resolver + // configuration over, and Netty's own pass calls resolve() -- *singular* -- so with a resolver + // that reports resolved addresses as unresolved, every candidate would be redirected again onto + // the resolver's first answer: the dead address, N times over. Multi-address fallback would + // silently do nothing, and the endpoint pinned onto the channel would name an address the + // channel is not connected to -- which is what the SSL engine's peer host and + // DefaultTopologyMonitor#savePort are then derived from. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup( + Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve()), + /* claimNothingIsResolved = */ true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – the reachable address was actually reached. This holds whichever candidate rotate() + // starts from, and it is precisely what fails when the clone re-resolves: the dead address is + // the resolver's first answer, so both attempts would land there and the connect would fail. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the hostname is expanded once, by ChannelFactory; the candidates are not re-resolved") + .containsExactly(HOSTNAME); + } + + @Test + public void should_resolve_and_connect_on_the_same_event_loop() throws InterruptedException { + // Resolution and channel registration must share the loop picked once per connect. Taking one + // loop for resolution and letting the registration pick another would advance the group's + // round-robin chooser twice per connect, parking every channel on half the loops with the + // default power-of-two chooser. The base's single-thread group would make this assertion + // vacuous, so use two loops -- on which the split behavior was deterministic. + DefaultEventLoopGroup twoLoops = new DefaultEventLoopGroup(2); + try { + when(nettyOptions.ioEventLoopGroup()).thenReturn(twoLoops); + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess( + channel -> + assertThat((Object) channel.eventLoop()) + .as("the channel must be registered on the loop resolution ran on") + .isSameAs(resolverGroup.resolverExecutor)); + } finally { + twoLoops.shutdownGracefully(0, 100, TimeUnit.MILLISECONDS).sync(); + } + } + + @Test + public void should_fail_future_when_resolver_throws_synchronously() { + // Given – a broken custom resolver that throws instead of returning a failed future. The throw + // happens inside an event-loop task, where nothing else would ever complete the connect future: + // nothing at this stage has a timeout, so before the blanket catch in resolveCandidates() this + // hung the connect attempt (and with it control-connection init) forever. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("broken resolver"); + installResolver(new ThrowingAddressResolverGroup(failure)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** A resolver whose every method throws, standing in for a broken third-party implementation. */ + private static class ThrowingAddressResolverGroup extends AddressResolverGroup { + + private final RuntimeException failure; + + ThrowingAddressResolverGroup(RuntimeException failure) { + this.failure = failure; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + throw failure; + } + + @Override + public boolean isResolved(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve( + SocketAddress address, Promise promise) { + throw failure; + } + + @Override + public Future> resolveAll(SocketAddress address) { + throw failure; + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + throw failure; + } + + @Override + public void close() { + // nothing to do + } + }; + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java new file mode 100644 index 00000000000..988b1ebe4f0 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies that a successfully connected {@link DriverChannel} carries an endpoint bound to the + * address the connection actually used, not the multi-address original. + * + *

Without this, a hostname shared by several nodes would let a later reconnect land on a + * different node while still being treated as the original {@code host_id}: {@code + * DefaultTopologyMonitor#buildNodeEndPoint} stores the channel's endpoint for the control node, and + * {@code ControlConnection} skips identity re-resolution for nodes that already have a host id. See + * {@link PinnableEndPoint}. + */ +public class ChannelFactoryPinnedEndPointTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryPinnedEndPointTest.class.getSimpleName() + "-unreachable"); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + @Test + public void should_pin_channel_endpoint_to_the_address_that_connected() { + // Given – an endpoint reporting a name, which the resolver expands to a dead address and the + // running local server. Whichever of the two the connection ends up on, the channel must carry + // that one. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, reachable))); + ChannelFactory factory = newChannelFactory(); + TestPinnableEndPoint endPoint = new TestPinnableEndPoint(HOSTNAME); + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture) + .isSuccess( + channel -> { + EndPoint channelEndPoint = channel.getEndPoint(); + // The channel resolves to the address it is actually connected to -- the name it was + // built from is gone from resolve(), which is what SSL engines and authenticators + // need. + assertThat(channelEndPoint.resolve()).isEqualTo(reachable); + // ...while still denoting the same node, so node lookups and metric names are stable. + assertThat(channelEndPoint).isEqualTo(endPoint); + assertThat(channelEndPoint.asMetricPrefix()).isEqualTo(endPoint.asMetricPrefix()); + }); + } + + @Test + public void should_leave_non_pinnable_endpoints_untouched() { + // A third-party EndPoint that does not implement PinnableEndPoint must reach the channel + // exactly + // as it was given, so existing implementations keep working. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess(channel -> assertThat(channel.getEndPoint()).isSameAs(SERVER_ADDRESS)); + } + + @Test + public void should_not_pin_to_an_unresolved_address() { + // Given – Bootstrap.disableResolver(), one of the paths where resolveCandidates hands the + // endpoint's own address straight back. For an endpoint that reports a name, the candidate is + // therefore still that name. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + doAnswer( + invocation -> { + ((Bootstrap) invocation.getArgument(0)).disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + List pinnedTo = new ArrayList<>(); + TestPinnableEndPoint endPoint = + new TestPinnableEndPoint(HOSTNAME) { + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + pinnedTo.add(resolvedAddress); + return super.pinTo(resolvedAddress); + } + }; + + // When – the connect itself cannot succeed against a name nothing resolves; what matters is + // what happened before it was attempted. + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – pinTo() is documented to take an already-resolved address. Pinning a name would freeze + // the endpoint on something that still re-expands on every connect, and for endpoints that stop + // consulting their own source once pinned, silence that source for good. + assertThatStage(channelFuture).isFailed(); + assertThat(pinnedTo).as("pinTo() must not be handed an unresolved address").isEmpty(); + } + + @Test + public void should_fail_future_when_pin_to_throws() { + // pinTo() runs in the continuation after resolution, whose exceptions CompletionStage + // swallows; a throwing implementation must fail the connect future rather than hang it. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Collections.singletonList(reachable))); + ChannelFactory factory = newChannelFactory(); + RuntimeException failure = new IllegalStateException("pinTo blew up"); + TestPinnableEndPoint endPoint = + new TestPinnableEndPoint(HOSTNAME) { + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + throw failure; + } + }; + + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** + * A {@link PinnableEndPoint} that can hold a pin to any {@link SocketAddress}, including the + * local-transport addresses these tests connect over (which {@code DefaultEndPoint} cannot). + * Identity is the unpinned address, so a pinned copy stays equal to the original — the contract + * {@link PinnableEndPoint} requires. + */ + private static class TestPinnableEndPoint implements PinnableEndPoint { + + private final SocketAddress address; + private final SocketAddress pinnedAddress; + + TestPinnableEndPoint(SocketAddress address) { + this(address, null); + } + + private TestPinnableEndPoint(SocketAddress address, SocketAddress pinnedAddress) { + this.address = address; + this.pinnedAddress = pinnedAddress; + } + + @NonNull + @Override + public SocketAddress resolve() { + return pinnedAddress != null ? pinnedAddress : address; + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + return new TestPinnableEndPoint(address, resolvedAddress); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + + @Override + public boolean equals(Object other) { + return (other instanceof TestPinnableEndPoint) + && address.equals(((TestPinnableEndPoint) other).address); + } + + @Override + public int hashCode() { + return Objects.hash(address); + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java index fceb8777904..658dbd8b15d 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -33,12 +34,21 @@ import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletionStage; import org.junit.Test; public class ChannelFactoryProtocolNegotiationTest extends ChannelFactoryTestBase { + /** A local address no server is bound to: connecting to it fails immediately. */ + private static final SocketAddress UNREACHABLE = + new LocalAddress( + ChannelFactoryProtocolNegotiationTest.class.getSimpleName() + "-unreachable"); + @Test public void should_succeed_if_version_specified_and_supported_by_server() { // Given @@ -280,6 +290,213 @@ public void should_fail_if_negotiation_finds_no_matching_version(int errorCode) }); } + @Test + public void should_not_try_next_address_of_identified_node_when_negotiation_exhausts_versions() { + // Given – an *identified* node (its host id is known, so every address its name expands to is + // that same node) whose name expands to two candidates: the same live server twice, so + // whichever the rotation picks first is irrelevant. The server rejects every protocol version. + mockNegotiationLadderDownToV3(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + exhaustNegotiationLadder(); + + // Then – the second candidate must not be attempted: for a node we have already identified, a + // protocol-version rejection is a property of the node, not of the address, so replaying the + // negotiation ladder against the remaining IPs would buy nothing. Checked before the future + // assertion so that on regression the stray frame is drained; leaving it unread would block the + // server's exchanger and hang the whole suite in tearDown() instead of failing this test. + assertThat(tryReadOutboundFrame(200)) + .as("second candidate must not be attempted after negotiation exhaustion") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("no other candidate should have been tried, so nothing to suppress") + .isEmpty(); + }); + } + + @Test + public void + should_try_next_address_of_unidentified_endpoint_when_negotiation_exhausts_versions() { + // Given – the same setup, but for an endpoint the driver has not identified yet: a contact + // point, before host ids have been read. Its name may well expand to addresses of *different* + // nodes, so a version rejection by the first says nothing about the second. + mockNegotiationLadderDownToV3(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + false); + + // The first candidate exhausts the ladder... + exhaustNegotiationLadder(); + // ...and the second is tried all the same, replaying the ladder from the top. Before this, + // resolve-contact-points=true made each address a separate node and ControlConnection advanced + // to the next one on exactly this error; collapsing a name into one node must not lose that. + exhaustNegotiationLadder(); + + // Then + assertThat(tryReadOutboundFrame(200)) + .as("the name expands to two addresses, so there is no third attempt") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .as("each candidate negotiates on its own, so this is the last one's ladder") + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("the first candidate's failure must still be reported") + .hasSize(1); + assertThat(e.getSuppressed()[0]) + .isInstanceOf(UnsupportedProtocolVersionException.class); + }); + } + + @Test + public void should_surface_the_node_wide_rejection_when_an_earlier_address_failed_on_transport() { + // Given – an identified node whose name expands to a dead address first and to the live server + // second, the server rejecting every protocol version. Every address of an identified node is + // that same node, so the rejection is a property of the node rather than of the address. + mockNegotiationLadderDownToV3(); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve()))); + ChannelFactory factory = newChannelFactory(); + // The order matters here, and only here: it is what makes the pass end with a *mixed* set of + // failures instead of the version rejection on its own. + factory.random = new KeepResolverOrder(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + // The dead address sends nothing, so the ladder below is the second candidate's. + exhaustNegotiationLadder(); + + // Then – the version rejection is what the caller must see, even though the pass also produced + // a + // transport failure that ChannelPool#handleError does not classify and would otherwise be + // preferred (see ChannelFactory#surfacedFailure: a fatal failure needs every address to agree). + // A node-wide failure is the exception to that rule: demoting it would turn the forced-down + // node + // that a single-address connect has always produced into a plain reconnect. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("the earlier address's transport failure should still be attached") + .hasSize(1); + }); + } + + /** Negotiation starts at V4 and has exactly one downgrade available, to V3. */ + private void mockNegotiationLadderDownToV3() { + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)) + .thenReturn(Optional.of(DefaultProtocolVersion.V3)); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V3)).thenReturn(Optional.empty()); + } + + /** + * Plays the server side of a full negotiation ladder against one candidate address: V4 rejected, + * downgrade retry with V3 rejected, i.e. no version left to try on that address. + */ + private void exhaustNegotiationLadder() { + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V3.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + } + + @Test + public void should_fail_future_when_downgrade_lookup_throws_in_connect_listener() { + // Given – a version registry that throws when the factory looks up the downgrade. The lookup + // runs inside the Netty connect listener, which swallows throwables: without the blanket catch + // in connectToAddress() the connect future would never complete and the attempt would hang. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("registry broken"); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)).thenThrow(failure); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + // Server does not support v4, which is what sends the factory to the downgrade lookup + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + /** * Depending on the Cassandra version, an "unsupported protocol" response can use different error * codes, so we test all of them. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java index ed6668a6c83..c1904c8f914 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java @@ -20,6 +20,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.ProtocolVersion; @@ -42,6 +44,7 @@ import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -53,9 +56,12 @@ import io.netty.channel.DefaultEventLoopGroup; import io.netty.channel.local.LocalChannel; import io.netty.channel.local.LocalServerChannel; +import io.netty.resolver.AddressResolverGroup; +import java.net.SocketAddress; import java.time.Duration; import java.util.Collections; import java.util.Optional; +import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Exchanger; import java.util.concurrent.TimeUnit; @@ -123,6 +129,9 @@ public void setup() throws InterruptedException { when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_SET_KEYSPACE_TIMEOUT)) .thenReturn(Duration.ofMillis(TIMEOUT_MILLIS)); when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_REQUESTS)).thenReturn(1); + // The reference.conf default; individual tests may lower it to exercise the cap. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(5); when(defaultProfile.getDuration(DefaultDriverOption.HEARTBEAT_INTERVAL)) .thenReturn(Duration.ofSeconds(30)); when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_CONNECT_TIMEOUT)) @@ -188,6 +197,55 @@ protected Frame readOutboundFrame() { return null; // never reached } + /** + * Like {@link #readOutboundFrame()}, but returns {@code null} instead of failing the test when no + * frame arrives within {@code timeoutMillis}. + * + *

Use this to assert that the client did not send another request. Unlike asserting via + * a failing read, it also drains a frame that does arrive: the server-side exchange in {@link + * ServerInitializer} has no timeout, so a stray unread frame would block the server event loop + * and hang the whole suite in {@link #tearDown()} instead of failing just the test. + */ + protected Frame tryReadOutboundFrame(long timeoutMillis) { + try { + return requestFrameExchanger.exchange(null, timeoutMillis, MILLISECONDS); + } catch (InterruptedException e) { + fail("unexpected interruption while waiting for outbound frame", e); + return null; // never reached + } catch (TimeoutException e) { + return null; + } + } + + /** + * A {@link Random} that turns {@link java.util.Collections#shuffle} into a no-op, so a test whose + * scenario depends on which address is tried first can rely on the order the resolver returned: + * shuffle swaps element {@code i-1} with {@code nextInt(i)}, and returning {@code i-1} swaps + * every element with itself. + * + *

Assign it to {@link ChannelFactory#random}, the injection point that exists for this. + */ + static final class KeepResolverOrder extends Random { + private static final long serialVersionUID = 1L; + + @Override + public int nextInt(int bound) { + return bound - 1; + } + } + + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ + protected void installResolver(AddressResolverGroup group) { + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(group); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + } + protected void writeInboundFrame(Frame requestFrame, Message response) { writeInboundFrame(requestFrame, response, requestFrame.protocolVersion); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java new file mode 100644 index 00000000000..06f5f636cba --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import edu.umd.cs.findbugs.annotations.Nullable; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * A stand-in for a user-supplied {@code AddressResolverGroup} (e.g. Netty's {@code + * DnsAddressResolverGroup}), installed the way a user would install one: through {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)}. + * + *

Records what it was asked to resolve, and answers with a fixed list of addresses so tests can + * assert that every one of them is tried, and in what order. + * + *

Implements {@link AddressResolver} directly rather than extending {@code + * AbstractAddressResolver} so it can hand back {@link LocalAddress}es — the unit tests connect over + * Netty's local transport, which is not reachable through an {@link InetSocketAddress}. + */ +class TestAddressResolverGroup extends AddressResolverGroup { + + /** Every address this group was asked to resolve, in order. */ + final List queried = new CopyOnWriteArrayList<>(); + + /** Whether a resolver was ever obtained from this group at all. */ + volatile boolean resolverRequested; + + /** The executor the last resolver was created for, i.e. the loop resolution runs on. */ + @Nullable volatile EventExecutor resolverExecutor; + + /** The addresses to answer with, or {@code null} to fail every lookup. */ + @Nullable private final List answer; + + /** + * Whether to claim that every address still needs resolving, even one that already carries an IP. + * A real resolver may do this to redirect traffic, and Netty honours it: {@code + * Bootstrap#doResolveAndConnect0} asks the resolver rather than testing the address itself. + */ + private final boolean claimNothingIsResolved; + + TestAddressResolverGroup(@Nullable List answer) { + this(answer, false); + } + + TestAddressResolverGroup(@Nullable List answer, boolean claimNothingIsResolved) { + this.answer = answer; + this.claimNothingIsResolved = claimNothingIsResolved; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + resolverRequested = true; + resolverExecutor = executor; + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + return true; + } + + @Override + public boolean isResolved(SocketAddress address) { + if (claimNothingIsResolved) { + return false; + } + // Only hostnames need resolving; anything else (including the local-transport addresses we + // hand back) is already usable. + return !(address instanceof InetSocketAddress) + || !((InetSocketAddress) address).isUnresolved(); + } + + @Override + public Future resolve(SocketAddress address) { + return resolve(address, executor.newPromise()); + } + + @Override + public Future resolve(SocketAddress address, Promise promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer.get(0)); + } + + @Override + public Future> resolveAll(SocketAddress address) { + return resolveAll(address, executor.newPromise()); + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer); + } + + @Override + public void close() { + // nothing to do + } + }; + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java index d51ee445da5..3b7c47535ea 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java @@ -26,6 +26,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.datastax.oss.driver.api.core.AllNodesFailedException; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; @@ -39,8 +41,11 @@ import com.datastax.oss.driver.internal.core.metadata.NodeStateEvent; import com.datastax.oss.driver.internal.core.metadata.TestNodeFactory; import com.datastax.oss.driver.internal.core.metadata.TopologyMonitor; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import java.net.ConnectException; import java.time.Duration; +import java.util.AbstractMap.SimpleEntry; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; @@ -964,4 +969,55 @@ public void should_not_call_getChannelNodeInfo_for_metadata_node_with_hostId() { factoryHelper.verifyNoMoreCalls(); } + + @Test + public void should_report_auth_failure_when_every_contact_point_failed_on_authentication() { + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>(node1, authError(node1)), + new SimpleEntry<>(node2, authError(node2)))); + + assertThat(ControlConnection.isAuthFailure(error)).isTrue(); + } + + @Test + public void should_not_report_auth_failure_when_an_address_failed_on_something_else() { + // One entry no longer means one address: ChannelFactory expands a contact-point hostname to + // every address it resolves to and reports one failure for the name, with the other addresses' + // failures suppressed. Reading only the top-level throwable would call this an authentication + // failure and tell the operator their credentials are wrong, when the name's other records are + // simply unreachable. + Throwable withUnreachableSibling = authError(node1); + withUnreachableSibling.addSuppressed(new ConnectException("connection refused")); + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of(new SimpleEntry<>(node1, withUnreachableSibling))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_not_report_auth_failure_when_a_node_failed_on_something_else() { + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>(node1, authError(node1)), + new SimpleEntry<>(node2, new ConnectException("connection refused")))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_not_report_auth_failure_for_a_throwable_with_no_per_node_breakdown() { + // Only an AllNodesFailedException carries the per-node errors this question is answered from. + // Anything else has nothing to say about the credentials, and answering yes would send an + // operator whose connection was refused off to check their authentication configuration. + assertThat(ControlConnection.isAuthFailure(new ConnectException("connection refused"))) + .isFalse(); + } + + private static Throwable authError(Node node) { + return new AuthenticationException(node.getEndPoint(), "mock authentication failure"); + } } From 1c512f2fa14a2232346d8a2df77793019250d3d5 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:16:10 +0200 Subject: [PATCH 5/9] fix: keep a node's metric identity stable across endpoint changes (DRIVER-201) Node metrics are named after the endpoint, so DefaultNode.setEndPoint() has to re-register them whenever those names change -- which is not the same question as whether this is a different node, and the old !equals() test got it wrong in both directions. It was too narrow: an unresolved hostname and the resolved address it maps to compare *equal* while their metric prefixes differ, which is exactly what happens when a contact-point node adopts the endpoint built from its system.local row. And too wide in the other direction is now possible too, since a pinned copy differs from its original only by an address that both equals() and the metric identity ignore by contract. The test is therefore asMetricPrefix() alone, plus the endpoint's runtime class, which keeps a node from staying on a plain fallback endpoint when a dynamic one carrying the same current address arrives. toString() is deliberately left out, even though the tagging MetricIdGenerator tags node metrics with it rather than with the prefix: it is not stable across equal instances, because DefaultEndPoint delegates it to InetSocketAddress, which renders InetAddress's *cached* hostName field, and DefaultSslEngineFactory populates that field while building an engine under the default allow-dns-reverse-lookup-san = true. Keying on it would report a difference for every node on every topology refresh, widening the clear/rebuild race below from "once per endpoint change" to "always". The cost of leaving it out: a tagging generator can keep reporting under an endpoint string the node no longer answers to, until something else changes the prefix. The pin is excluded from toString() as well, or DefaultTopologyMonitor#buildNodeEndPoint returning the control channel's pinned endpoint for the system.local row would silently retag node metrics on every refresh and orphan the old series. The comparison lives in PinnableEndPoint#sameIdentity, next to the contract it encodes, because ControlConnection asks the same question when the control channel adopts a node's endpoint. The node also adopts the newest endpoint instance even when it compares equal, since a pinned copy carries the address every subsequent connection will use. Finally, the rebuild order is clear, then swap, then build. Dropwizard and MicroProfile do not remember the ids they registered under; their clearMetrics() recomputes each one from the node's endpoint as it stands at that moment. The previous order -- swap, build, clear -- therefore deleted exactly the series the new updater had just registered and left the old ones behind with nothing writing to them. That ordering is upstream's, but it used to be reached only when the endpoints compared unequal; keying the rebuild on metric identity brings the ordinary contact-point transition onto the same path. The pre-existing pin test was vacuous: a mocked context yields NoopNodeMetricUpdater, for which the rebuild is skipped entirely. Both tests now stub MetricsFactory, and the ordering test drives a real MetricRegistry through a hostname-to-IP rename; it was proven to fail under the old order. Co-Authored-By: Claude Opus 5 (1M context) --- .../internal/core/metadata/DefaultNode.java | 108 +++++++- .../core/metrics/AbstractMetricUpdater.java | 30 ++ .../core/metrics/NodeMetricUpdater.java | 21 +- .../core/metadata/DefaultNodeTest.java | 256 ++++++++++++++++++ .../DropwizardNodeMetricUpdaterTest.java | 120 ++++++++ 5 files changed, 525 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java index 1b09c26ce16..16da0750f8f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java @@ -102,15 +102,105 @@ public EndPoint getEndPoint() { } public void setEndPoint(@NonNull EndPoint newEndPoint, @NonNull InternalDriverContext context) { - if (!newEndPoint.equals(endPoint)) { - endPoint = newEndPoint; - // metricUpdater is transient, so it can be null on deserialized nodes. - NodeMetricUpdater previousMetricUpdater = metricUpdater; - if (previousMetricUpdater != null - && !(previousMetricUpdater instanceof NoopNodeMetricUpdater)) { - metricUpdater = context.getMetricsFactory().newNodeUpdater(this); - previousMetricUpdater.clearMetrics(); - } + // Nothing downstream can tell the two instances apart, so keep the one already held. Not merely + // an optimization: the instance this node holds may carry a reverse-DNS name cached on its + // InetAddress by an earlier TLS handshake (DefaultSslEngineFactory calls getHostName() under + // the + // default advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true), and every full + // topology refresh mints a brand-new endpoint over a brand-new InetSocketAddress for every + // node. + // Adopting each one unconditionally would throw that name away and make the next connection to + // the node repeat the blocking reverse lookup on a Netty I/O loop -- once per node per refresh + // instead of once per node per session, during exactly the refresh-then-reconnect storms this + // feature exists for. + // + // Deliberately not equals(): see PinnableEndPoint#sameIdentity, which is also what + // ControlConnection uses when it decides whether the control channel should adopt a node's + // endpoint. + if (PinnableEndPoint.sameIdentity(newEndPoint, endPoint)) { + return; + } + + // Metrics are registered under names derived from the endpoint, so they have to be + // re-registered + // whenever those names change -- which is not the same question as whether this is a different + // node. It is narrower in one direction: a PinnableEndPoint copy differs from the original only + // by the address it is pinned to, and both equals() and the metric identity ignore that by + // contract (see PinnableEndPoint). And it is wider in the other: an unresolved hostname and the + // resolved address it maps to compare *equal* (see DefaultEndPoint#equals) while their metric + // prefixes differ, which is exactly what happens when a contact-point node adopts the endpoint + // built from its system.local row. + // + // asMetricPrefix() alone, deliberately, even though the tagging MetricIdGenerator tags metrics + // with the endpoint's toString() rather than its prefix. toString() cannot be used as an + // identity key because it is not stable across equal instances: DefaultEndPoint delegates it to + // InetSocketAddress, which renders InetAddress's *cached* hostName field, and that field is + // populated the first time anything calls getHostName(). DefaultSslEngineFactory does exactly + // that while building an engine, under the default advanced.ssl-engine-factory + // .allow-dns-reverse-lookup-san = true -- on the very instance this node holds, since an + // already-resolved endpoint is passed through unchanged by resolveCandidates() and pin(). So + // after the first channel this node's endpoint renders as "host/1.2.3.4:9042" while the one the + // next refresh decodes from system.peers renders as "/1.2.3.4:9042", and keying on that would + // clear and re-register every node's metrics on every topology refresh -- widening the + // clear/rebuild race described below from "once per endpoint change" to "always". + // + // The cost of leaving it out: a tagging generator can keep reporting under an endpoint string + // the node no longer answers to, until something else changes the prefix. + boolean differentMetricIdentity = + !newEndPoint.asMetricPrefix().equals(endPoint.asMetricPrefix()); + // metricUpdater is transient, so it can be null on deserialized nodes. + NodeMetricUpdater previousMetricUpdater = metricUpdater; + boolean rebuildMetricUpdater = + differentMetricIdentity + && previousMetricUpdater != null + && !(previousMetricUpdater instanceof NoopNodeMetricUpdater); + + // Clearing comes *before* the swap. Dropwizard and MicroProfile do not remember the ids they + // registered under; clearMetrics() recomputes each one from this node's current endpoint (see + // DropwizardMetricUpdater#clearMetrics and MetricIdGenerator#nodeMetricId). Clearing after the + // swap would therefore delete the series the new updater had just registered and leave the old + // ones behind, under a name nothing writes to any more. Micrometer removes the Meter instances + // it holds and does not care either way. + // + // The three steps are not atomic with respect to concurrent metric writes: metricUpdater is + // volatile and read from I/O threads, so a write landing between the clear and the rebuild + // goes through the updater that was just cleared, and Dropwizard re-registers on demand + // (getOrCreateCounterFor -> registry.counter(getMetricId(m))). That resurrects one series, + // named from whichever endpoint this node holds at that instant. + // + // Note the window is narrow but its effect is not transient: the resurrected metric is cached + // in the old updater's map, and ChannelFactory snapshots node.getMetricUpdater() once per + // connection and hands it to the traffic meters, which hold it for the channel's life. So a + // mark that lands here keeps reporting under the old endpoint's name until every channel open + // at that moment has been recycled. Nothing throws -- registry.counter() is get-or-create -- + // and the request path re-reads getMetricUpdater() per request, so the misreporting is + // confined to the byte counters. Closing it properly means having clearMetrics() remove the + // ids it registered under rather than recomputing them from the current endpoint, which is a + // change to every metrics implementation and would also fix a second problem: two nodes can + // briefly share a metric prefix (a control node's endpoint is its contact point's), and then + // this clear deletes the series the other node just registered. + if (rebuildMetricUpdater) { + previousMetricUpdater.clearMetrics(); + } + + // Adopt the newest instance even when it compares equal: a pinned copy carries the address + // every + // subsequent connection to this node will use, so refusing it would freeze the node on the + // first + // address it ever connected to, even after the control connection moved to another one and told + // us about it. (The early return above lets through exactly the instances that differ in that + // address, or in metric identity, or in kind.) + endPoint = newEndPoint; + + // And building comes *after* it: the updaters register every enabled metric from their + // constructor, deriving the names from the endpoint this node holds at that moment. + if (rebuildMetricUpdater) { + NodeMetricUpdater newMetricUpdater = context.getMetricsFactory().newNodeUpdater(this); + // Carry over any pending metrics expiration before publishing the replacement: the factories + // arm and cancel it through node.getMetricUpdater(), so from here on they would only ever + // reach the new one, leaving the old one's timer pending on an object nothing refers to. + newMetricUpdater.adoptExpirationFrom(previousMetricUpdater); + metricUpdater = newMetricUpdater; } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java index 3d7dc50a7c0..6000510b0e9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java @@ -167,6 +167,36 @@ protected void cancelMetricsExpirationTimeout() { } } + /** + * Moves a pending expiration from the updater being replaced onto this one. See {@link + * NodeMetricUpdater#adoptExpirationFrom}. + * + *

Re-armed rather than handed over as-is, so the replacement's own {@code + * startMetricsExpirationTimeout()} runs and the countdown belongs to the object whose metrics it + * will clear. That restarts the clock; expiry is a coarse, hour-scale cleanup and the node has to + * stay down for the whole period either way, so the reset is not worth carrying the original + * deadline around for. + * + *

Only a timeout that was still pending is carried over. {@link Timeout#cancel()} returns + * {@code false} when the task has already run or was already cancelled, and the first of those is + * reachable: the task is {@code clearMetrics()} followed by {@code + * cancelMetricsExpirationTimeout()}, so an endpoint change landing between the two finds a + * non-null reference to an expired timeout. Re-arming on that would start a fresh hour-long + * countdown and expire the replacement's freshly registered metrics, even though this node's + * metrics had already been cleared once. + */ + public void adoptExpirationFrom(NodeMetricUpdater previous) { + if (!(previous instanceof AbstractMetricUpdater)) { + return; + } + Timeout pending = + ((AbstractMetricUpdater) previous).metricsExpirationTimeoutRef.getAndSet(null); + if (pending == null || !pending.cancel()) { + return; + } + startMetricsExpirationTimeout(); + } + protected Timeout newTimeout() { return context .getNettyOptions() diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java index 93d003f0a03..af84782c989 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java @@ -19,4 +19,23 @@ import com.datastax.oss.driver.api.core.metrics.NodeMetric; -public interface NodeMetricUpdater extends MetricUpdater {} +public interface NodeMetricUpdater extends MetricUpdater { + + /** + * Takes over the metrics-expiration countdown from the updater this one is replacing, when a node + * rebuilds its updater after its endpoint changed. + * + *

Without this the countdown is simply lost. It is armed and cancelled through {@code + * node.getMetricUpdater()} -- by the metrics factories, on node state events -- so once a node + * has swapped in a replacement, the cancel that a later UP event triggers reaches the new updater + * and finds nothing, while the old updater's timer is still pending on an object nothing else + * refers to. Both halves of that are wrong: the replacement never expires, because a node that is + * already down will not produce another DOWN event to arm it, and the orphan eventually fires + * {@link #clearMetrics()} on names it recomputes from whatever endpoint the node holds by then. + * + *

Implementations that do not expire metrics can ignore this. + */ + default void adoptExpirationFrom(NodeMetricUpdater previous) { + // nothing to hand over + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java index 6a53fe3e433..f844f9d2805 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java @@ -18,10 +18,31 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.codahale.metrics.MetricRegistry; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverConfig; +import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; +import com.datastax.oss.driver.api.core.metrics.NodeMetric; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.MockedDriverContextFactory; +import com.datastax.oss.driver.internal.core.metrics.AbstractMetricUpdater; +import com.datastax.oss.driver.internal.core.metrics.DefaultMetricIdGenerator; +import com.datastax.oss.driver.internal.core.metrics.DropwizardNodeMetricUpdater; +import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; +import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Collections; +import java.util.Set; import java.util.UUID; import org.junit.Test; @@ -55,4 +76,239 @@ public void should_have_expected_string_representation_if_hostid_is_null() { "Node(endPoint=localhost/127.0.0.1:9042, hostId=null, hashCode=%x)", node.hashCode()); assertThat(node.toString()).isEqualTo(expected); } + + @Test + public void should_adopt_a_newer_endpoint_that_only_differs_by_its_pinned_address() { + // A PinnableEndPoint copy compares equal to the original -- pinnedAddress is excluded from + // equals() by contract, so that a pinned copy still denotes the same node. setEndPoint() must + // therefore not use equals() to decide whether to adopt it: the pinned address is the one every + // subsequent connection to this node will use, so refusing the newer instance would freeze the + // node on the first address it ever connected to, even after the control connection has moved + // and told us about it. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + DefaultNode node = new DefaultNode(endPoint, context); + + EndPoint pinnedToFirst = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)); + node.setEndPoint(pinnedToFirst, context); + assertThat(node.getEndPoint()).isSameAs(pinnedToFirst); + + EndPoint pinnedToSecond = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.3", 9042)); + // Same node by equals(), different pinned address. + assertThat(pinnedToSecond).isEqualTo(pinnedToFirst); + node.setEndPoint(pinnedToSecond, context); + + assertThat(node.getEndPoint()).isSameAs(pinnedToSecond); + assertThat(node.getEndPoint().resolve()).isEqualTo(new InetSocketAddress("127.0.0.3", 9042)); + } + + @Test + public void should_keep_the_endpoint_instance_it_already_holds_when_nothing_would_differ() { + // Every full topology refresh mints a fresh endpoint over a fresh InetSocketAddress for every + // node, and for an unchanged node it denotes exactly what this one already does. Adopting it + // would discard the reverse-DNS name that an earlier TLS handshake cached on the InetAddress + // this + // node holds (DefaultSslEngineFactory calls getHostName() under the default + // allow-dns-reverse-lookup-san = true), so the next connection would repeat that blocking + // lookup + // on an I/O event loop -- once per node per refresh instead of once per node per session. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + EndPoint held = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + DefaultNode node = new DefaultNode(held, context); + + EndPoint equivalent = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(equivalent).isNotSameAs(held); + node.setEndPoint(equivalent, context); + + assertThat(node.getEndPoint()).isSameAs(held); + } + + @Test + public void should_adopt_an_endpoint_of_a_different_kind_that_resolves_to_the_same_address() { + // A dynamic endpoint (ClientRoutesEndPoint) resolves through its topology monitor on every + // call, + // so it is not interchangeable with a static one that happens to point at the same address + // today. + // The early return must not keep the static one in that case. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + EndPoint stat1c = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + DefaultNode node = new DefaultNode(stat1c, context); + + EndPoint dynamic = new RenamingEndPoint("/127.0.0.1:9042"); + assertThat(dynamic.resolve()).isEqualTo(stat1c.resolve()); + assertThat(dynamic.asMetricPrefix()).isEqualTo(stat1c.asMetricPrefix()); + + node.setEndPoint(dynamic, context); + + assertThat(node.getEndPoint()).isSameAs(dynamic); + } + + @Test + public void should_not_rebuild_the_metric_updater_for_a_pin_only_change() { + // A pinned copy is identified exactly like the original -- same asMetricPrefix(), same + // toString() -- so rebuilding would clear and re-register metrics under identical names, and + // reset their values along the way. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + DefaultNode node = new DefaultNode(endPoint, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint( + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)), context); + + assertThat(node.getMetricUpdater()).isSameAs(first); + verify(first, never()).clearMetrics(); + } + + @Test + public void should_not_rebuild_the_metric_updater_when_only_the_endpoints_string_form_differs() { + // toString() is not usable as an identity key because it is not stable across instances that + // denote the same thing: DefaultEndPoint delegates it to InetSocketAddress, which renders + // InetAddress's cached hostName field, and that field is filled in the first time anything + // calls getHostName() -- which DefaultSslEngineFactory does, on this node's own endpoint + // instance, under the default allow-dns-reverse-lookup-san = true. Keying on it would clear and + // re-register every node's metrics on every topology refresh, since each refresh decodes a + // fresh endpoint whose hostName has not been filled in yet. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + + EndPoint beforeLookup = new RenamingEndPoint("/127.0.0.1:9042"); + EndPoint afterLookup = new RenamingEndPoint("host.example.com/127.0.0.1:9042"); + assertThat(beforeLookup.asMetricPrefix()).isEqualTo(afterLookup.asMetricPrefix()); + assertThat(beforeLookup.toString()).isNotEqualTo(afterLookup.toString()); + + DefaultNode node = new DefaultNode(beforeLookup, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint(afterLookup, context); + + assertThat(node.getMetricUpdater()).isSameAs(first); + verify(first, never()).clearMetrics(); + } + + /** An endpoint that always reports the same metric prefix but renders differently. */ + private static class RenamingEndPoint implements EndPoint { + private final String stringForm; + + RenamingEndPoint(String stringForm) { + this.stringForm = stringForm; + } + + @NonNull + @Override + public SocketAddress resolve() { + return new InetSocketAddress("127.0.0.1", 9042); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "127_0_0_1:9042"; + } + + @Override + public String toString() { + return stringForm; + } + } + + @Test + public void should_rebuild_the_metric_updater_when_an_equal_endpoint_renames_the_metrics() { + // An unresolved hostname and the address it resolves to compare *equal* (see + // DefaultEndPoint#equals) but do not produce the same metric prefix. That is exactly what + // happens when a contact-point node adopts the endpoint built from its system.local row, so + // deciding on equals() alone would leave the node's metrics registered under the hostname while + // asMetricPrefix() had moved on to the IP. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + + EndPoint asHostname = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + EndPoint asAddress = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(asHostname).isEqualTo(asAddress); + assertThat(asHostname.asMetricPrefix()).isNotEqualTo(asAddress.asMetricPrefix()); + + DefaultNode node = new DefaultNode(asHostname, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint(asAddress, context); + + assertThat(node.getMetricUpdater()).isSameAs(second); + verify(first).clearMetrics(); + } + + @Test + public void should_rebuild_the_metric_updater_without_wiping_the_metrics_it_registers() { + // The two tests above use mocks, which cannot see *when* clearMetrics() runs relative to the + // endpoint swap -- and that order is what decides whether the rebuild works. Dropwizard and + // MicroProfile do not remember the ids they registered under: clearMetrics() recomputes each + // one + // from the node's endpoint as it stands at that moment. Clearing after the swap therefore + // removes exactly the series the new updater has just registered, and leaves the old ones in + // the + // registry with nothing writing to them. So this one drives a real registry. + MetricRegistry registry = new MetricRegistry(); + NodeMetric metric = DefaultNodeMetric.UNSENT_REQUESTS; + InternalDriverContext context = dropwizardContext(registry, Collections.singleton(metric)); + + EndPoint asHostname = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + EndPoint asAddress = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + String underHostname = "s.nodes." + asHostname.asMetricPrefix() + '.' + metric.getPath(); + String underAddress = "s.nodes." + asAddress.asMetricPrefix() + '.' + metric.getPath(); + assertThat(underHostname).isNotEqualTo(underAddress); + + DefaultNode node = new DefaultNode(asHostname, context); + assertThat(registry.getNames()).containsExactly(underHostname); + + node.setEndPoint(asAddress, context); + + assertThat(registry.getNames()).containsExactly(underAddress); + } + + /** A context wired to a real Dropwizard registry, enough for {@link DefaultNode} to use it. */ + private static InternalDriverContext dropwizardContext( + MetricRegistry registry, Set enabledMetrics) { + InternalDriverContext context = mock(InternalDriverContext.class); + DriverConfig config = mock(DriverConfig.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + when(context.getSessionName()).thenReturn("s"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(profile.getString(DefaultDriverOption.METRICS_ID_GENERATOR_PREFIX, "")).thenReturn(""); + // Built outside the when(): the generator's constructor reads back from the context, and + // Mockito rejects a nested call on a stubbing that is still open. + DefaultMetricIdGenerator idGenerator = new DefaultMetricIdGenerator(context); + when(context.getMetricIdGenerator()).thenReturn(idGenerator); + + MetricsFactory metricsFactory = mock(MetricsFactory.class); + when(context.getMetricsFactory()).thenReturn(metricsFactory); + // Built on demand rather than up front: an updater registers its metrics from its constructor, + // under the names the node's endpoint yields at that point. + when(metricsFactory.newNodeUpdater(any())) + .thenAnswer( + invocation -> + new DropwizardNodeMetricUpdater( + invocation.getArgument(0), context, enabledMetrics, registry)); + return context; + } + + /** A context whose only stubbed behaviour is the metrics factory {@code DefaultNode} asks for. */ + private static InternalDriverContext contextWith(MetricsFactory metricsFactory) { + InternalDriverContext context = mock(InternalDriverContext.class); + when(context.getMetricsFactory()).thenReturn(metricsFactory); + return context; + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java index ccc42a7027d..c31a77324e9 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java @@ -18,8 +18,11 @@ package com.datastax.oss.driver.internal.core.metrics; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -33,13 +36,18 @@ import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; import com.datastax.oss.driver.api.core.metrics.NodeMetric; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.util.LoggerTest; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; import java.time.Duration; import java.util.Collections; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.junit.Test; import org.junit.runner.RunWith; @@ -152,6 +160,118 @@ protected void initializeHdrTimer( verify(logger.appender, timeout(500).times(0)).doAppend(logger.loggingEventCaptor.capture()); } + @Test + public void should_adopt_a_pending_expiration_from_the_updater_it_replaces() { + // given – two updaters for the same node, as DefaultNode.setEndPoint builds when an endpoint + // change renames the metrics, and an expiration already armed on the one being replaced. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout pendingTimeout = mock(Timeout.class); + Timeout adoptedTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(pendingTimeout, adoptedTimeout); + // Still pending, i.e. the countdown had not run yet: that is what makes it worth carrying over. + when(pendingTimeout.cancel()).thenReturn(true); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + // when + replacement.adoptExpirationFrom(previous); + + // then – the countdown is cancelled on the object nothing refers to any more and re-armed on + // the + // one whose metrics it will clear. Without this the expiration is simply lost: it is armed and + // cancelled through node.getMetricUpdater(), so a node that is already down never produces + // another DOWN event to arm the replacement, while the orphan timer still fires clearMetrics() + // on names recomputed from whatever endpoint the node holds by then. + verify(pendingTimeout).cancel(); + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + // and – there is nothing left to hand over a second time. + replacement.adoptExpirationFrom(previous); + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_not_re_arm_an_expiration_that_has_already_fired() { + // The timer task is clearMetrics() followed by cancelMetricsExpirationTimeout(), so an endpoint + // change landing between the two finds a non-null reference to a timeout that has already run. + // Re-arming on that would start a fresh expire-after countdown and wipe the replacement's newly + // registered metrics an hour later, even though this node's metrics were already cleared once. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout expiredTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(expiredTimeout); + // Timeout.cancel() returns false once the task has run (or was already cancelled). + when(expiredTimeout.cancel()).thenReturn(false); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + replacement.adoptExpirationFrom(previous); + + verify(timer, times(1)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + /** A node updater with metric registration stubbed out: only its expiration behavior matters. */ + private static DropwizardNodeMetricUpdater newNodeUpdater( + Node node, InternalDriverContext context) { + return new DropwizardNodeMetricUpdater( + node, + context, + Collections.singleton(DefaultNodeMetric.CQL_MESSAGES), + new MetricRegistry()) { + @Override + protected void initializeGauge( + NodeMetric metric, DriverExecutionProfile profile, Supplier supplier) { + // do nothing + } + + @Override + protected void initializeCounter(NodeMetric metric, DriverExecutionProfile profile) { + // do nothing + } + + @Override + protected void initializeHdrTimer( + NodeMetric metric, + DriverExecutionProfile profile, + DriverOption highestLatency, + DriverOption significantDigits, + DriverOption interval) { + // do nothing + } + }; + } + @DataProvider public static Object[][] acceptableEvictionTimes() { return new Object[][] { From 06d2b7f7f96dba582d7e099a8f0d945491219efc Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Tue, 11 Aug 2026 00:30:40 +0200 Subject: [PATCH 6/9] fix: identify the control node through a connect hook in the candidate loop (DRIVER-201) ChannelFactory tries every address a contact-point hostname resolves to while it opens a channel, but the node's identity was read afterwards, over the channel that won -- by then the remaining candidates are gone. ControlConnection advanced its query plan on that failure, and since a contact-point hostname is now a single Node, that wrote off the whole hostname on the strength of one of its addresses. With a single contact point and the default reconnect-on-init=false, session initialization failed outright, and a rebuilt session failed the same way every time while a healthy address sat unused. The identity read now happens while the factory still holds the remaining candidates, through a caller-supplied hook. DriverChannelOptions gains an internal ConnectHook (precedent for a behavioral member there: eventCallback) plus a timeout; after protocol initialization succeeds on a candidate, ChannelFactory invokes the hook and treats a rejection, a synchronous throw or a timeout as a per-candidate failure: the channel is closed and the loop advances to the endpoint's next address, exactly like an init failure. ControlConnection arms the hook only for a node whose host id is unknown -- contact points, the one case with something to learn. The hook runs TopologyMonitor.getChannelNodeInfo, so a custom monitor's identity read is honored; it rejects a node that reports no host id, and channels what it read straight into a per-attempt holder. Once the connect completes, the captured NodeInfo is registered without a second read, after asserting it came from the winning channel; a miss (a ChannelFactory subclass that skips the hook) falls back to a direct read. The options are built fresh per attempt, because the holder is stateful and overlapping connect chains are reachable: the initial connect() runs outside Reconnection, and reconnectNow() checks only initWasCalled. Pool connections and reconnects to identified nodes carry no hook and send exactly the bytes they sent before. REGISTER moves out of the init handshake to keep its ordering property: identity is validated before the channel registers for events. ChannelFactory sends it through AdminRequestHandler after the hook accepts, with the same init-query timeout; a registration failure is a per-candidate failure, as it was as an init step, and the CLIENT_ROUTES_CHANGE rejection keeps its clear message. The one visible cost: the window in which a live channel is not yet registered for events grows by the hook's round trip. The wire cost is unchanged from reading identity after the connect: one "SELECT * FROM system.local WHERE key='local'" per contact-point connection, now inside the attempt instead of after it. Init itself ends at the cluster-name check (or SET_KEYSPACE), byte-identical to the pre-multi-address exchange, which is what ProtocolVersionMixedClusterIT pins. Two consequences are visible. A contact point that exhausts every address on identity failures now fires controlConnectionFailed and is marked DOWN pre-init, the treatment connect-phase failures already get; and AllNodesFailedException reports one entry per contact point, with the per-address failures attached as suppressed exceptions. One pre-existing exposure stays pre-existing rather than closing: ChannelFactory imprints the cluster name, product type and negotiated protocol version on init success, so a candidate the hook then rejects has already imprinted -- like every other channel abandoned after init (a node turned IGNORED, a close during resolve). The values are properties of the cluster that answered on that address, so this is identical to the behavior before this series. Co-Authored-By: Claude Opus 5 (1M context) Co-Authored-By: Claude Fable 5 --- .../adminrequest/AdminRequestHandler.java | 30 +- .../internal/core/channel/ChannelFactory.java | 328 ++++++++++- .../internal/core/channel/ConnectHook.java | 62 ++ .../core/channel/DriverChannelOptions.java | 49 +- .../core/channel/ProtocolInitHandler.java | 27 - .../core/control/ControlConnection.java | 332 +++++++++-- .../core/metadata/DefaultTopologyMonitor.java | 79 ++- .../ChannelFactoryConnectHookTest.java | 549 ++++++++++++++++++ .../core/channel/ProtocolInitHandlerTest.java | 120 ++-- .../core/control/ControlConnectionTest.java | 373 ++++++++++++ .../metadata/DefaultTopologyMonitorTest.java | 71 +++ 11 files changed, 1814 insertions(+), 206 deletions(-) create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java index 5078428c21a..02496d68ae1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java @@ -30,8 +30,9 @@ import com.datastax.oss.protocol.internal.Message; import com.datastax.oss.protocol.internal.ProtocolConstants; import com.datastax.oss.protocol.internal.request.Query; +import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.query.QueryOptions; -import com.datastax.oss.protocol.internal.response.Result; +import com.datastax.oss.protocol.internal.response.Ready; import com.datastax.oss.protocol.internal.response.result.Prepared; import com.datastax.oss.protocol.internal.response.result.Rows; import io.netty.util.concurrent.Future; @@ -67,6 +68,24 @@ public static AdminRequestHandler call( com.datastax.oss.protocol.internal.response.result.Void.class); } + /** + * Registers this connection for the given protocol events, as {@link + * com.datastax.oss.protocol.internal.request.Register REGISTER} used to be sent as the last step + * of protocol initialization. + */ + public static AdminRequestHandler register( + DriverChannel channel, List eventTypes, Duration timeout, String logPrefix) { + return new AdminRequestHandler<>( + channel, + true, + new Register(eventTypes), + Frame.NO_PAYLOAD, + timeout, + logPrefix, + "register for events " + eventTypes, + Ready.class); + } + public static AdminRequestHandler query( DriverChannel channel, String query, @@ -98,7 +117,7 @@ public static AdminRequestHandler query( private final Duration timeout; private final String logPrefix; private final String debugString; - private final Class expectedResponseType; + private final Class expectedResponseType; protected final CompletableFuture result = new CompletableFuture<>(); // This is only ever accessed on the channel's event loop, so it doesn't need to be volatile @@ -112,7 +131,7 @@ protected AdminRequestHandler( Duration timeout, String logPrefix, String debugString, - Class expectedResponseType) { + Class expectedResponseType) { this.channel = channel; this.shouldPreAcquireId = shouldPreAcquireId; this.message = message; @@ -190,8 +209,9 @@ public void onResponse(Frame responseFrame) { @SuppressWarnings("unchecked") ResultT result = (ResultT) ByteBuffer.wrap(prepared.preparedQueryId); setFinalResult(result); - } else if (expectedResponseType - == com.datastax.oss.protocol.internal.response.result.Void.class) { + } else if (expectedResponseType == com.datastax.oss.protocol.internal.response.result.Void.class + || expectedResponseType == Ready.class) { + // Neither carries a payload: a schema change or a REGISTER acknowledgement. setFinalResult(null); } else { setFinalError(new AssertionError("Unhandled response type" + expectedResponseType)); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index c37245103b1..7e696fe7d7c 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -31,12 +31,15 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.connection.ConnectionInitException; import com.datastax.oss.driver.api.core.context.DriverContext; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeShardingInfo; import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; import com.datastax.oss.driver.api.core.metrics.DefaultSessionMetric; +import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; +import com.datastax.oss.driver.internal.core.adminrequest.UnexpectedResponseException; import com.datastax.oss.driver.internal.core.config.typesafe.TypesafeDriverConfig; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.NettyOptions; @@ -52,6 +55,8 @@ import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import com.datastax.oss.protocol.internal.Message; +import com.datastax.oss.protocol.internal.ProtocolConstants; import com.datastax.oss.protocol.internal.ProtocolFeatures; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -63,6 +68,7 @@ import io.netty.resolver.AddressResolver; import io.netty.resolver.AddressResolverGroup; import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.ScheduledFuture; import java.io.IOException; import java.net.Inet6Address; import java.net.InetAddress; @@ -71,6 +77,7 @@ import java.net.ServerSocket; import java.net.SocketAddress; import java.net.UnknownHostException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.IdentityHashMap; @@ -83,6 +90,7 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import net.jcip.annotations.ThreadSafe; @@ -954,9 +962,19 @@ private void tryNextCandidate( *

An {@link AuthenticationException} is deliberately absent too, even for an identified node: * see {@link #tryNextCandidate} on why authentication must advance, and {@link #shuffleAndLimit} * for the cap that bounds what wrong credentials can cost. + * + *

An {@link UnsupportedEventTypeException} is node-wide, for a contact point as much as + * for an identified node, and unlike every other case here it does not depend on {@code + * nodeIsIdentified}: it says the server does not know an event type the driver asked to register + * for, which is a property of the software running there, not of the record that reached it. + * Replaying it against the remaining addresses can only fail the same way, and each replay costs + * a full TCP connect plus the STARTUP/AUTH/cluster-name handshake and the connect hook's round + * trip. Stopping at the first one restores what this rejection cost while REGISTER was an init + * step, which was one failed connect per contact point. */ private static boolean isNodeWideFailure(Throwable error, boolean nodeIsIdentified) { - return error instanceof UnsupportedProtocolVersionException && nodeIsIdentified; + return (error instanceof UnsupportedProtocolVersionException && nodeIsIdentified) + || error instanceof UnsupportedEventTypeException; } /** @@ -1129,33 +1147,11 @@ private void connectToAddress( DriverChannel driverChannel = new DriverChannel( endPoint, channel, context.getWriteCoalescer(), currentVersion); - // If this is the first successful connection, remember the protocol version and - // cluster name for future connections. - if (isNegotiating) { - ChannelFactory.this.protocolVersion = currentVersion; - } - if (ChannelFactory.this.clusterName == null) { - ChannelFactory.this.clusterName = driverChannel.getClusterName(); - } - Map> supportedOptions = driverChannel.getOptions(); - if (ChannelFactory.this.productType == null && supportedOptions != null) { - List productTypes = supportedOptions.get("PRODUCT_TYPE"); - String productType = - productTypes != null && !productTypes.isEmpty() - ? productTypes.get(0) - : UNKNOWN_PRODUCT_TYPE; - ChannelFactory.this.productType = productType; - DriverConfig driverConfig = context.getConfig(); - if (driverConfig instanceof TypesafeDriverConfig - && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { - ((TypesafeDriverConfig) driverConfig) - .overrideDefaults( - ImmutableMap.of( - DefaultDriverOption.REQUEST_CONSISTENCY, - ConsistencyLevel.LOCAL_QUORUM.name())); - } - } - perAddressFuture.complete(driverChannel); + finishCandidate( + driverChannel, + options, + perAddressFuture, + () -> latchNegotiatedState(driverChannel, currentVersion, isNegotiating)); } else { Throwable error = connectFuture.cause(); if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { @@ -1211,6 +1207,282 @@ private void connectToAddress( } } + /** + * Remembers what the first accepted connection negotiated, so later connections skip the + * negotiation: the protocol version, the cluster name to check others against, and the server's + * product type (which for Cloud also lowers the default consistency level). + * + *

Run only once a candidate has been accepted, not as soon as its transport connect and + * init handshake succeed. Init is no longer the last word on a candidate: the connect hook can + * reject it (the control connection's identity read does, for a node with no {@code host_id}), + * and REGISTER, which used to be the final init step, now runs after that hook. A candidate the + * driver is about to throw away must not leave its cluster name latched here -- a stale DNS + * record pointing at a foreign cluster would otherwise make every subsequent connection fail its + * cluster-name check, which {@code ChannelPool} turns into an irreversible forced-down node. + */ + private void latchNegotiatedState( + DriverChannel driverChannel, ProtocolVersion currentVersion, boolean isNegotiating) { + if (isNegotiating) { + this.protocolVersion = currentVersion; + } + if (this.clusterName == null) { + this.clusterName = driverChannel.getClusterName(); + } + Map> supportedOptions = driverChannel.getOptions(); + if (this.productType == null && supportedOptions != null) { + List productTypes = supportedOptions.get("PRODUCT_TYPE"); + String productType = + productTypes != null && !productTypes.isEmpty() + ? productTypes.get(0) + : UNKNOWN_PRODUCT_TYPE; + this.productType = productType; + DriverConfig driverConfig = context.getConfig(); + if (driverConfig instanceof TypesafeDriverConfig + && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { + ((TypesafeDriverConfig) driverConfig) + .overrideDefaults( + ImmutableMap.of( + DefaultDriverOption.REQUEST_CONSISTENCY, ConsistencyLevel.LOCAL_QUORUM.name())); + } + } + } + + /** + * The tail of a candidate attempt, once transport connect and protocol initialization have both + * succeeded: runs the caller's {@link ConnectHook} (if any), then registers for protocol events + * (if requested), and only then completes the candidate's future. + * + *

Both steps happen while this attempt still holds the endpoint's remaining addresses, so a + * failure in either is a per-candidate failure: the channel is force-closed on the spot and + * {@link #tryNextCandidate} advances to the next address. + * + *

REGISTER used to be the last protocol-init step; it moved behind the hook so that a channel + * the hook is about to reject never registers for events. The window in which a live channel is + * not yet registered grows by the hook's round trip -- the same order of cost as the init step it + * follows. + */ + private void finishCandidate( + DriverChannel driverChannel, + DriverChannelOptions options, + CompletableFuture perAddressFuture, + Runnable onAccepted) { + if (options.connectHook == null) { + registerForEvents(driverChannel, options, perAddressFuture, onAccepted); + return; + } + CompletionStage vetted; + try { + vetted = options.connectHook.onConnect(driverChannel); + } catch (Throwable t) { + // A synchronous throw is a rejection, like an exceptional stage. Blanket-caught: this runs + // inside a Netty listener that swallows throwables, so a caller-supplied callback leaking + // one would otherwise leave the attempt hanging forever. + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook rejected the channel", t)); + return; + } + if (vetted == null) { + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook returned a null stage", null)); + return; + } + // The hook's contract says its stage eventually completes, but only the driver can make that + // true: a wedged hook (a topology monitor whose own timeout is broken, say) would otherwise + // hang the whole connect attempt, and with it control-connection init or a reconnect. + // + // Unless the timeout is zero or negative, which every other consumer of a driver timeout option + // reads as "no timeout" (see AdminRequestHandler#onWriteComplete). Scheduling it anyway would + // fire on the next event-loop turn, before any round trip can complete, and abandon every + // candidate of every contact point -- so an operator who disabled the control-connection + // timeout + // would find that the session cannot initialize at all. + ScheduledFuture hookTimeout; + try { + hookTimeout = + (options.connectHookTimeout == null || options.connectHookTimeout.toNanos() <= 0) + ? null + : driverChannel + .eventLoop() + .schedule( + () -> + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException( + "Connect hook timed out after " + options.connectHookTimeout, + null)), + options.connectHookTimeout.toNanos(), + TimeUnit.NANOSECONDS); + } catch (Throwable t) { + // An event loop shutting down rejects the task. Fail the candidate rather than run the hook + // with nothing bounding it: this method is called from a Netty listener that swallows + // throwables, so the attempt would otherwise hang (see connectToAddress's invariant). + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Could not schedule the connect hook timeout", t)); + return; + } + vetted.whenComplete( + (aVoid, error) -> { + try { + if (hookTimeout != null) { + hookTimeout.cancel(false); + } + if (error != null) { + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook rejected the channel", error)); + } else { + registerForEvents(driverChannel, options, perAddressFuture, onAccepted); + } + } catch (Throwable t) { + // Blanket-caught, as everywhere else in this class: nobody consumes the stage this + // callback returns, and the timeout that would have failed the candidate has just been + // cancelled, so anything escaping here -- registerForEvents' config read, for instance + // -- would leave perAddressFuture uncompleted forever. + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException( + "Unexpected error after the connect hook accepted the channel", t)); + } + }); + } + + /** + * Sends the REGISTER request when the options ask for protocol events, then completes the + * candidate. A registration failure is a per-candidate failure, exactly as it was when REGISTER + * was a protocol-init step. + */ + private void registerForEvents( + DriverChannel driverChannel, + DriverChannelOptions options, + CompletableFuture perAddressFuture, + Runnable onAccepted) { + if (options.eventTypes.isEmpty()) { + completeCandidate(driverChannel, perAddressFuture, onAccepted); + return; + } + Duration timeout = + context + .getConfig() + .getDefaultProfile() + .getDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT); + AdminRequestHandler.register(driverChannel, options.eventTypes, timeout, logPrefix) + .start() + .whenComplete( + (aVoid, error) -> { + try { + if (error != null) { + abandonCandidate( + driverChannel, perAddressFuture, translateRegisterFailure(error)); + } else { + completeCandidate(driverChannel, perAddressFuture, onAccepted); + } + } catch (Throwable t) { + // Blanket-caught, as everywhere else in this class: nobody consumes the stage this + // callback returns, and by this point no timeout is left to fail the candidate, so + // anything escaping -- translateRegisterFailure's casts, a forceClose() on an event + // loop that is shutting down -- would leave perAddressFuture uncompleted forever + // and hang the connect attempt (see connectToAddress's invariant). + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Unexpected error after REGISTER", t)); + } + }); + } + + /** + * Gives the one REGISTER rejection with a known cause a message that names it: the server not + * knowing the {@code CLIENT_ROUTES_CHANGE} event type. This translation lived in the init handler + * when REGISTER was an init step, and exists so that the caller + * (ClientRoutesTopologyMonitor.init()) reports a clear error instead of silently degrading. + */ + private static Throwable translateRegisterFailure(Throwable error) { + if (error instanceof UnexpectedResponseException) { + Message response = ((UnexpectedResponseException) error).message; + if (response instanceof com.datastax.oss.protocol.internal.response.Error) { + com.datastax.oss.protocol.internal.response.Error protocolError = + (com.datastax.oss.protocol.internal.response.Error) response; + if (protocolError.code == ProtocolConstants.ErrorCode.PROTOCOL_ERROR + && protocolError.message.contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)) { + return new UnsupportedEventTypeException( + "Server does not support CLIENT_ROUTES_CHANGE event " + + "(requires ScyllaDB Enterprise >= 2026.1). " + + "Either upgrade the server or remove the client routes configuration.", + error); + } + } + } + return error; + } + + /** + * A REGISTER rejection that is a property of the server -- it does not know an event type + * the driver asked for -- rather than of the address that was dialled. + * + *

A {@link ConnectionInitException}, so that callers which branch on the type (see {@link + * #surfacedFailure}, and {@code ClientRoutesTopologyMonitor#init}, which reports the message) + * treat it exactly as they treated the same rejection when REGISTER was an init step. The subtype + * exists only so {@link #isNodeWideFailure} can recognise it. + */ + @VisibleForTesting + static class UnsupportedEventTypeException extends ConnectionInitException { + UnsupportedEventTypeException(String message, Throwable cause) { + super(message, cause); + } + } + + /** + * Completes the candidate's future, closing the channel if something else -- the hook timeout, + * the only concurrent completer -- got there first: nothing else would close it, and an + * unpublished channel nobody holds is a leak. + * + *

Winning that race is also what makes this candidate the accepted one, and therefore the only + * one entitled to record what it negotiated (see {@link #latchNegotiatedState}). + */ + private static void completeCandidate( + DriverChannel driverChannel, + CompletableFuture perAddressFuture, + Runnable onAccepted) { + if (perAddressFuture.complete(driverChannel)) { + onAccepted.run(); + } else { + driverChannel.forceClose(); + } + } + + /** + * Closes a candidate channel that will not be used and fails its future -- unless that channel + * has meanwhile been handed to the caller, in which case it is theirs and must be left alone. + */ + private static void abandonCandidate( + DriverChannel driverChannel, + CompletableFuture perAddressFuture, + Throwable error) { + // The hook timeout and the hook's own completion race, and the hook's stage may complete off + // the + // channel's event loop -- the contract allows it, and a custom TopologyMonitor behind the + // control + // connection's hook is free to -- so cancel(false) can lose to a timeout task that has already + // started running. The loser of that race must not close a channel the winner published: + // completeExceptionally() is a silent no-op on an already *successful* future, so closing + // unconditionally would leave the caller owning a dead channel and no error to explain it. + // Failing it ourselves, or finding it already failed (by another abandonCandidate, or by the + // connect listener), is what makes the channel ours to close; forceClose is idempotent. + if (perAddressFuture.completeExceptionally(error) + || perAddressFuture.isCompletedExceptionally()) { + driverChannel.forceClose(); + } + } + /** * Binds {@code endPoint} to the address a connection is being opened to, when the implementation * supports it. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java new file mode 100644 index 00000000000..fb09f153dc3 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import java.util.concurrent.CompletionStage; + +/** + * A caller-supplied step that runs against every candidate channel a {@link ChannelFactory#connect} + * attempt opens, after protocol initialization succeeds and before the attempt is considered + * successful. + * + *

Its position is what makes it useful: a connect attempt may try several addresses when the + * endpoint's name resolves to more than one, and the hook runs while the factory still holds the + * remaining ones. Completing the returned stage exceptionally (or throwing synchronously) rejects + * the candidate -- the factory closes the channel and moves on to the endpoint's next address -- so + * a caller can impose its own acceptance criteria on a channel, per address, without losing the + * fallback. The control connection uses this to read {@code system.local} and refuse a channel + * whose node cannot identify itself, channeling what it read straight into its own state (see + * {@code ControlConnection}). + * + *

Contract: + * + *

    + *
  • invoked at most once per candidate channel, and never concurrently within one connect + * attempt (candidates are tried serially); + *
  • invoked on the channel's event loop: implementations must not block, and anything heavier + * than an asynchronous request on the channel itself should hop to another thread; + *
  • the returned stage must eventually complete; the factory bounds it with {@link + * DriverChannelOptions#connectHookTimeout} and rejects the candidate when it expires; + *
  • only channel-scoped resources may be touched: the channel is not published to the caller + * yet, and a rejected or timed-out candidate is closed by the factory. + *
+ * + *

When the options also request protocol events ({@link DriverChannelOptions#eventTypes}), the + * {@code REGISTER} request is sent after the hook completes successfully, so a channel that is + * about to be rejected never registers for events. + */ +public interface ConnectHook { + + /** + * Vets a candidate channel that completed protocol initialization. + * + * @return a stage that completes normally to accept the channel, or exceptionally to reject it + * and make the connect attempt move on to the endpoint's next address. + */ + CompletionStage onConnect(DriverChannel channel); +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java index 378fd2dc0b8..7f8d768153d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; +import java.time.Duration; import java.util.Collections; import java.util.List; import net.jcip.annotations.Immutable; @@ -54,17 +55,41 @@ public static Builder builder() { */ public final boolean reportConfig; + /** + * A step the caller runs against every candidate channel after protocol initialization, with the + * power to reject it while {@link ChannelFactory} still holds the endpoint's other addresses, or + * {@code null} if the caller has no vetting to do. Precedent for a behavioral member here: {@link + * #eventCallback}. + * + *

The control connection supplies one when connecting to a node whose {@code host_id} is not + * yet known -- a contact point, the one case with something to learn -- to read {@code + * system.local} and refuse a channel whose node cannot identify itself. + * + * @see ConnectHook + */ + public final ConnectHook connectHook; + + /** + * How long the factory waits for {@link #connectHook}'s stage before treating the candidate as + * rejected. Never null when {@link #connectHook} is set. + */ + public final Duration connectHookTimeout; + private DriverChannelOptions( CqlIdentifier keyspace, List eventTypes, EventCallback eventCallback, String ownerLogPrefix, - boolean reportConfig) { + boolean reportConfig, + ConnectHook connectHook, + Duration connectHookTimeout) { this.keyspace = keyspace; this.eventTypes = eventTypes; this.eventCallback = eventCallback; this.ownerLogPrefix = ownerLogPrefix; this.reportConfig = reportConfig; + this.connectHook = connectHook; + this.connectHookTimeout = connectHookTimeout; } public static class Builder { @@ -73,6 +98,8 @@ public static class Builder { private EventCallback eventCallback = null; private String ownerLogPrefix = null; private boolean reportConfig = false; + private ConnectHook connectHook = null; + private Duration connectHookTimeout = null; public Builder withKeyspace(CqlIdentifier keyspace) { this.keyspace = keyspace; @@ -100,9 +127,27 @@ public Builder reportConfig(boolean reportConfig) { return this; } + /** + * Arms a step that vets every candidate channel after protocol initialization, bounded by the + * given timeout. See {@link ConnectHook} for the contract. + */ + public Builder withConnectHook(ConnectHook connectHook, Duration connectHookTimeout) { + Preconditions.checkNotNull(connectHook); + Preconditions.checkNotNull(connectHookTimeout); + this.connectHook = connectHook; + this.connectHookTimeout = connectHookTimeout; + return this; + } + public DriverChannelOptions build() { return new DriverChannelOptions( - keyspace, eventTypes, eventCallback, ownerLogPrefix, reportConfig); + keyspace, + eventTypes, + eventCallback, + ownerLogPrefix, + reportConfig, + connectHook, + connectHookTimeout); } } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java index dd7630a6530..042006ea9a6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java @@ -52,7 +52,6 @@ import com.datastax.oss.protocol.internal.request.AuthResponse; import com.datastax.oss.protocol.internal.request.Options; import com.datastax.oss.protocol.internal.request.Query; -import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.AuthChallenge; import com.datastax.oss.protocol.internal.response.AuthSuccess; @@ -159,7 +158,6 @@ private enum Step { GET_CLUSTER_NAME, SET_KEYSPACE, AUTH_RESPONSE, - REGISTER, } private class InitRequest extends ChannelHandlerRequest { @@ -170,12 +168,10 @@ private class InitRequest extends ChannelHandlerRequest { private Message request; private Authenticator authenticator; private ByteBuffer authResponseToken; - private final List registerEventTypes; InitRequest(ChannelHandlerContext ctx) { super(ctx, timeoutMillis); this.step = querySupportedOptions ? Step.OPTIONS : Step.STARTUP; - this.registerEventTypes = options.eventTypes; } @Override @@ -206,8 +202,6 @@ Message getRequest() { return request = new Query("USE " + options.keyspace.asCql(false)); case AUTH_RESPONSE: return request = new AuthResponse(authResponseToken); - case REGISTER: - return request = new Register(registerEventTypes); default: throw new AssertionError("unhandled step: " + step); } @@ -330,21 +324,11 @@ void onResponse(Message response) { if (options.keyspace != null) { step = Step.SET_KEYSPACE; send(); - } else if (!registerEventTypes.isEmpty()) { - step = Step.REGISTER; - send(); } else { setConnectSuccess(); } } } else if (step == Step.SET_KEYSPACE && response instanceof SetKeyspace) { - if (!registerEventTypes.isEmpty()) { - step = Step.REGISTER; - send(); - } else { - setConnectSuccess(); - } - } else if (step == Step.REGISTER && response instanceof Ready) { setConnectSuccess(); } else if (response instanceof Error) { Error error = (Error) response; @@ -366,17 +350,6 @@ void onResponse(Message response) { } else if (step == Step.SET_KEYSPACE && error.code == ProtocolConstants.ErrorCode.INVALID) { fail(new InvalidKeyspaceException(error.message)); - } else if (step == Step.REGISTER - && error.code == ErrorCode.PROTOCOL_ERROR - && error.message.contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)) { - // The server rejected CLIENT_ROUTES_CHANGE as an unknown event type. - // Fail the connection so that the caller (ClientRoutesTopologyMonitor.init()) - // gets a clear error instead of silently degrading. - fail( - "Server does not support CLIENT_ROUTES_CHANGE event " - + "(requires ScyllaDB Enterprise >= 2026.1). " - + "Either upgrade the server or remove the client routes configuration.", - null); } else { failOnUnexpected(error); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java index 51cddf0b728..1081dfd2538 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java @@ -34,11 +34,14 @@ import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.ClientRoutesUpdateEvent; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.internal.core.metadata.DefaultTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.DistanceEvent; import com.datastax.oss.driver.internal.core.metadata.MetadataManager; +import com.datastax.oss.driver.internal.core.metadata.NodeInfo; import com.datastax.oss.driver.internal.core.metadata.NodeStateEvent; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metadata.TopologyEvent; import com.datastax.oss.driver.internal.core.util.Loggers; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; @@ -56,6 +59,7 @@ import com.datastax.oss.protocol.internal.response.event.TopologyChangeEvent; import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.util.concurrent.EventExecutor; +import java.time.Duration; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Collection; @@ -277,7 +281,9 @@ private class SingleThreaded { private boolean closeWasCalled; private final ReconnectionPolicy reconnectionPolicy; private final Reconnection reconnection; - private DriverChannelOptions channelOptions; + // Computed once in init() and kept; the options themselves are built fresh for every connect + // attempt (see buildChannelOptions), so this is the only part of them that lives here. + private ImmutableList eventTypes; private volatile ControlNodeState controlNodeState = ControlNodeState.NONE; // The last events received for each node private final Map lastNodeDistance = new WeakHashMap<>(); @@ -321,15 +327,8 @@ private void init( try { boolean listenClientRoutesEvents = context.getTopologyMonitor() instanceof ClientRoutesTopologyMonitor; - ImmutableList eventTypes = - buildEventTypes(listenToClusterEvents, listenClientRoutesEvents); + this.eventTypes = buildEventTypes(listenToClusterEvents, listenClientRoutesEvents); LOG.debug("[{}] Initializing with event types {}", logPrefix, eventTypes); - channelOptions = - DriverChannelOptions.builder() - .withEvents(eventTypes, ControlConnection.this) - .withOwnerLogPrefix(logPrefix + "|control") - .reportConfig(true) - .build(); Queue nodes = context.getLoadBalancingPolicyWrapper().newControlReconnectionQueryPlan(); @@ -394,19 +393,25 @@ private void connect( onFailure.accept(AllNodesFailedException.fromErrors(errors)); } else { LOG.debug("[{}] Trying to establish a connection to {}", logPrefix, node); + NodeInfoHolder capturedNodeInfo = new NodeInfoHolder(); context .getChannelFactory() - .connect(node, channelOptions) + .connect(node, buildChannelOptions(node, capturedNodeInfo)) .whenCompleteAsync( (channel, error) -> { try { - NodeDistance lastDistance = lastNodeDistance.get(node); - NodeState lastState = lastNodeState.get(node); + String exclusion = exclusionReason(node); if (error != null) { if (closeWasCalled || initFuture.isCancelled()) { onSuccess.run(); // abort, we don't really care about the result } else { - if (error instanceof AuthenticationException) { + // isAuthOnly, not a bare instanceof: ChannelFactory reports one failure + // per contact point with the other addresses' failures attached as + // suppressed, and it deliberately surfaces an authentication failure over + // transport ones. A name whose records failed [refused, refused, auth] is + // not an authentication problem, and logging it as one would hide that two + // thirds of the deployment is unreachable. + if (isAuthOnly(error)) { Loggers.warnWithException( LOG, "[{}] Authentication error", logPrefix, error); } else { @@ -440,22 +445,12 @@ private void connect( channel); channel.forceClose(); onSuccess.run(); - } else if (lastDistance == NodeDistance.IGNORED) { + } else if (exclusion != null) { LOG.debug( - "[{}] New channel opened ({}) but node became ignored, " - + "closing and trying next node", + "[{}] New channel opened ({}) but {}, closing and trying next node", logPrefix, - channel); - channel.forceClose(); - connect(nodes, errors, onSuccess, onFailure); - } else if (lastNodeState.containsKey(node) - && (lastState == null /*(removed)*/ - || lastState == NodeState.FORCED_DOWN)) { - LOG.debug( - "[{}] New channel opened ({}) but node was removed or forced down, " - + "closing and trying next node", - logPrefix, - channel); + channel, + exclusion); channel.forceClose(); connect(nodes, errors, onSuccess, onFailure); } else { @@ -470,7 +465,7 @@ private void connect( previousChannel); previousChannel.forceClose(); } - resolveChannelNodeIfNeeded(channel, (DefaultNode) node) + resolveChannelNodeIfNeeded(channel, (DefaultNode) node, capturedNodeInfo) .whenCompleteAsync( (resolvedNode, fetchError) -> { if (fetchError != null) { @@ -501,19 +496,58 @@ private void connect( new Exception("Channel closed during endpoint resolve"))); connect(nodes, newErrors, onSuccess, onFailure); } else { - controlNodeState = new ControlNodeState(resolvedNode, null); - context - .getEventBus() - .fire(ChannelEvent.channelOpened(resolvedNode)); - channel - .closeFuture() - .addListener( - f -> - adminExecutor - .submit( - () -> onChannelClosed(channel, resolvedNode)) - .addListener(UncaughtExceptions::log)); - onSuccess.run(); + // The guards above ran against the node the query plan offered. + // For a contact point appended by the reconnection fallback that + // is an ephemeral instance which is never the subject of a + // distance or state event, so it is never a key in either map and + // those guards cannot have seen anything. Only now, once the + // handshake has said which node actually answered, is there + // something to ask about -- and asking matters, because nothing + // downstream will: an unchanged distance fires no event, so a + // control connection parked on an excluded node stays there. + String resolvedExclusion = exclusionReason(resolvedNode); + if (resolvedExclusion != null) { + LOG.debug( + "[{}] Channel {} turned out to be {}, which {}; " + + "closing and trying next node", + logPrefix, + channel, + resolvedNode, + resolvedExclusion); + controlNodeState = ControlNodeState.NONE; + // Null out before forceClose(), as above, so that + // onChannelClosed() does not start a redundant reconnection on + // top of the connect() retry below. + ControlConnection.this.channel = null; + channel.forceClose(); + // Recorded, so that exhausting the plan this way reports why + // rather than a bare NoNodeAvailableException. No + // controlConnectionFailed event though: nothing failed to + // connect, the node is simply not one we may use -- same as the + // pre-handshake exclusion branch above. + List> newErrors = + (errors == null) ? new ArrayList<>() : errors; + newErrors.add( + new SimpleEntry<>( + resolvedNode, + new IllegalStateException(resolvedExclusion))); + connect(nodes, newErrors, onSuccess, onFailure); + } else { + controlNodeState = new ControlNodeState(resolvedNode, null); + context + .getEventBus() + .fire(ChannelEvent.channelOpened(resolvedNode)); + channel + .closeFuture() + .addListener( + f -> + adminExecutor + .submit( + () -> + onChannelClosed(channel, resolvedNode)) + .addListener(UncaughtExceptions::log)); + onSuccess.run(); + } } }, adminExecutor); @@ -531,28 +565,123 @@ private void connect( } /** - * Resolves the identity of the node at the other end of the channel. For contact point nodes - * (no hostId), queries system.local and registers a new metadata node. For nodes that already - * have a hostId, returns the node as-is. + * Why {@code candidate} must not be used for the control connection -- the load balancing + * policy has excluded it, or a topology event has -- or {@code null} if nothing rules it out. + * + *

Both maps are keyed on the {@link Node} instance and filled only from events, so a node + * that has never been the subject of one is simply absent, and absent means "nothing known + * against it" rather than "fine". That distinction is why this is asked twice per connect: once + * about the node the query plan offered, and again about the node the handshake proved is at + * the other end, which for a contact point is a different instance. */ - private CompletionStage resolveChannelNodeIfNeeded( - DriverChannel channel, DefaultNode node) { - if (node.getHostId() != null) { - return CompletableFuture.completedFuture(node); + private String exclusionReason(Node candidate) { + if (lastNodeDistance.get(candidate) == NodeDistance.IGNORED) { + return "node became ignored"; + } + if (lastNodeState.containsKey(candidate)) { + NodeState state = lastNodeState.get(candidate); + if (state == null /*(removed)*/ || state == NodeState.FORCED_DOWN) { + return "node was removed or forced down"; + } + } + return null; + } + + /** + * Options for one connect attempt against one query-plan node. Built fresh per attempt rather + * than cached: an attempt against an unidentified node carries a stateful holder that the + * connect hook fills, and overlapping connect chains are reachable -- the initial {@code + * connect()} runs outside {@code Reconnection} (which serializes only its own attempts), and + * {@code reconnectNow()} checks only {@code initWasCalled} -- so nothing stateful may be shared + * between attempts. + */ + private DriverChannelOptions buildChannelOptions(Node node, NodeInfoHolder capturedNodeInfo) { + DriverChannelOptions.Builder builder = + DriverChannelOptions.builder() + .withEvents(eventTypes, ControlConnection.this) + .withOwnerLogPrefix(logPrefix + "|control") + .reportConfig(true); + if (node.getHostId() == null) { + // A contact point: the driver does not yet know which node answers at each of its + // addresses, so the identity read happens through the connect hook, inside the factory's + // candidate loop, where an address that cannot identify itself is rejected while the + // hostname's other addresses are still on hand. Read after the connect instead, the + // candidates are already gone and a failure writes off the whole plan entry. + Duration hookTimeout = + config.getDefaultProfile().getDuration(DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT); + builder.withConnectHook( + channel -> readChannelNodeInfo(channel, capturedNodeInfo), hookTimeout); } + return builder.build(); + } + + /** + * The connect hook of a contact-point attempt: reads which node answered and channels it + * straight into the attempt's holder, rejecting the candidate when the node cannot identify + * itself. + * + *

Runs on the channel's event loop and touches no {@code SingleThreaded} state: the holder + * is the only thing written, and it is read back on the admin thread only after the connect + * completes. + */ + private CompletionStage readChannelNodeInfo( + DriverChannel channel, NodeInfoHolder capturedNodeInfo) { return context .getTopologyMonitor() .getChannelNodeInfo(channel) - .thenComposeAsync( + .thenAccept( nodeInfo -> { - EndPoint resolvedEp = nodeInfo.getEndPoint(); - if (resolvedEp != null && !resolvedEp.equals(channel.getEndPoint())) { - channel.setEndPoint(resolvedEp); - LOG.debug("[{}] Control channel endpoint upgraded to {}", logPrefix, resolvedEp); - } - return context.getMetadataManager().registerNode(nodeInfo); - }, - adminExecutor); + // Mirrors DefaultTopologyMonitor's own precondition, so that a custom monitor + // cannot smuggle a null past registerNode: rejecting here costs one address, while + // failing in registerNode later would cost the whole plan entry. + Objects.requireNonNull( + nodeInfo.getHostId(), + "Node info is missing its host id; the node may still be bootstrapping"); + capturedNodeInfo.set(channel, nodeInfo); + }); + } + + /** + * Resolves the identity of the node at the other end of the channel. For nodes that already + * have a hostId, returns the node as-is. For a contact point, the connect hook has already read + * {@code system.local} and captured the result (see {@link #readChannelNodeInfo}); this + * registers a new metadata node from it. + */ + private CompletionStage resolveChannelNodeIfNeeded( + DriverChannel channel, DefaultNode node, NodeInfoHolder capturedNodeInfo) { + if (node.getHostId() != null) { + return CompletableFuture.completedFuture(node); + } + NodeInfo captured = capturedNodeInfo.getFor(channel); + // The pairing with the channel is asserted rather than assumed, and a miss falls back to a + // direct read: a ChannelFactory subclass that does not run the connect hook still gets a + // functioning control connection (and the mocked factories in the unit tests exercise this + // same path). The fallback costs one extra round trip, on that path only. + CompletionStage nodeInfoFuture = + (captured != null) + ? CompletableFuture.completedFuture(captured) + : context.getTopologyMonitor().getChannelNodeInfo(channel); + return nodeInfoFuture.thenComposeAsync( + nodeInfo -> { + EndPoint resolvedEp = nodeInfo.getEndPoint(); + EndPoint channelEp = channel.getEndPoint(); + // The channel adopts the node's endpoint so that everything reading it afterwards -- + // DefaultTopologyMonitor's localEndPoint on the next refresh, refreshNode's + // control-node + // check, OptionalLocalDcHelper's endpoint fallback -- sees the same instance the node + // holds, rather than the contact point this connection happened to come up through. + // + // The same test as DefaultNode#setEndPoint, and deliberately not equals(): this is + // exactly the mixed unresolved-vs-resolved case (see PinnableEndPoint#sameIdentity). + if (resolvedEp != null + && resolvedEp != channelEp + && !PinnableEndPoint.sameIdentity(resolvedEp, channelEp)) { + channel.setEndPoint(resolvedEp); + LOG.debug("[{}] Control channel endpoint upgraded to {}", logPrefix, resolvedEp); + } + return context.getMetadataManager().registerNode(nodeInfo); + }, + adminExecutor); } private void onSuccessfulReconnect() { @@ -688,10 +817,44 @@ private boolean isControlNode(Node eventNode) { && eventNode.getHostId().equals(state.current.getHostId())) { return true; } - if (state.current == null - && state.pending != null - && Objects.equals(eventNode.getEndPoint(), state.pending.getEndPoint())) { - return true; + if (state.current == null && state.pending != null) { + // Resolution is still in flight, so there is no host id to compare yet and the endpoint is + // all there is to go on. The channel's own endpoint is what to compare against: unlike the + // pending node's, ChannelFactory has bound it to the one address the connection actually + // went to. + DriverChannel pendingChannel = ControlConnection.this.channel; + if (pendingChannel == null) { + return false; + } + EndPoint eventEndPoint = eventNode.getEndPoint(); + EndPoint channelEndPoint = pendingChannel.getEndPoint(); + // Two lookup-free comparisons, because neither shape is covered by the other. + // + // resolve() settles it when both sides hold a concrete address: the event carries a + // metadata + // node whose endpoint is a resolved IP, and the channel's is pinned to the IP it reached. + // This is the case the plain hostname contact point hits, and comparing resolve() results + // rather than the endpoints keeps DefaultEndPoint#equals -- which resolves the unresolved + // side of a mixed comparison, i.e. a blocking DNS lookup on the admin thread, on an + // arbitrary single address (issue #1006) -- off a path that runs for every distance or + // state + // event arriving during a control connect. + if (Objects.equals(eventEndPoint.resolve(), channelEndPoint.resolve())) { + return true; + } + // But resolve() cannot settle it when the endpoint's current address is a *name*, which is + // the permanent state of an SNI proxy address, of a client route, and of anything a custom + // AddressTranslator hands over unresolved: the event node resolves to that unresolved name + // while the channel resolves to the IP it was pinned to, and an InetSocketAddress carrying + // an InetAddress never equals one that does not. For those endpoints identity does not key + // on the address at all -- SniEndPoint on proxy + serverName, ClientRoutesEndPoint on the + // host id, both of which a Cloud contact point and its metadata node share -- so equals() + // answers exactly the right question, and does so without resolving anything. Every + // implementation except DefaultEndPoint qualifies; that one is excluded precisely because + // its equals() is the one that would resolve. + return eventEndPoint.getClass() == channelEndPoint.getClass() + && !(eventEndPoint instanceof DefaultEndPoint) + && eventEndPoint.equals(channelEndPoint); } return false; } @@ -787,7 +950,8 @@ static boolean isAuthFailure(Throwable error) { } /** Whether {@code error} and every failure attached to it are authentication failures. */ - private static boolean isAuthOnly(Throwable error) { + @VisibleForTesting + static boolean isAuthOnly(Throwable error) { if (!(error instanceof AuthenticationException)) { return false; } @@ -799,6 +963,50 @@ private static boolean isAuthOnly(Throwable error) { return true; } + /** + * What one contact-point connect attempt learned about the node that answered: filled by the + * connect hook on the channel's event loop, read back on the admin thread once the connect + * completes. One instance per attempt -- overlapping attempts must not share it, which is why + * {@code DriverChannelOptions} are built per attempt. + */ + @VisibleForTesting + static final class NodeInfoHolder { + + /** + * The two values are published as one reference so that a reader can never see one candidate's + * node info paired with another's channel. Two separate volatile fields would not do: the + * factory's candidate loop can leave a stranded hook behind -- an attempt abandoned on the hook + * timeout, whose {@code system.local} response then arrives anyway -- and its late write would + * land in between the accepted candidate's two writes. The admin thread reading in that window + * would take the rejected candidate's node info as the accepted channel's, and the control + * connection would then register the wrong host id and endpoint for the node it is talking to. + * + *

With the pair atomic, that late write merely makes {@link #getFor} miss, which falls back + * to reading {@code system.local} again on the channel that is actually open. + */ + private volatile Capture capture; + + void set(DriverChannel channel, NodeInfo nodeInfo) { + this.capture = new Capture(channel, nodeInfo); + } + + /** The captured info if it came from {@code channel}: the pairing is asserted, not assumed. */ + NodeInfo getFor(DriverChannel channel) { + Capture current = this.capture; + return (current != null && current.channel == channel) ? current.nodeInfo : null; + } + + private static final class Capture { + final DriverChannel channel; + final NodeInfo nodeInfo; + + Capture(DriverChannel channel, NodeInfo nodeInfo) { + this.channel = channel; + this.nodeInfo = nodeInfo; + } + } + } + /** * Immutable snapshot of the control node state. Reads from any thread see a consistent pair of * (current, pending) via a single volatile read of the enclosing reference. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java index 5a82bfe2c86..f3761733c32 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java @@ -352,22 +352,25 @@ public CompletionStage getChannelNodeInfo(DriverChannel channel) { } EndPoint localEndPoint = channel.getEndPoint(); return query(channel, buildQuery(localColumns, "system.local", "key='local'")) - .thenApply( - result -> { - if (localColumns == null && !result.getColumnNames().isEmpty()) { - localColumns = - intersectWithNeeded(result.getColumnNames(), LOCAL_COLUMNS_OF_INTEREST); - } - Iterator iterator = result.iterator(); - if (!iterator.hasNext()) { - throw new IllegalStateException( - "Expected a row in system.local for node info resolution, got empty result"); - } - AdminRow localRow = iterator.next(); - InetSocketAddress broadcastRpcAddress = - getBroadcastRpcAddress(localRow, localEndPoint); - return nodeInfoBuilder(localRow, broadcastRpcAddress, localEndPoint).build(); - }); + .thenApply(result -> toLocalNodeInfo(result, localEndPoint)); + } + + /** + * Decodes the single {@code system.local} row of {@code result} into a {@link NodeInfo}, warming + * the local column cache from it on the way. + */ + private NodeInfo toLocalNodeInfo(AdminResult result, EndPoint localEndPoint) { + if (localColumns == null && !result.getColumnNames().isEmpty()) { + localColumns = intersectWithNeeded(result.getColumnNames(), LOCAL_COLUMNS_OF_INTEREST); + } + Iterator iterator = result.iterator(); + if (!iterator.hasNext()) { + throw new IllegalStateException( + "Expected a row in system.local for node info resolution, got empty result"); + } + AdminRow localRow = iterator.next(); + InetSocketAddress broadcastRpcAddress = getBroadcastRpcAddress(localRow, localEndPoint); + return nodeInfoBuilder(localRow, broadcastRpcAddress, localEndPoint).build(); } @Override @@ -659,8 +662,52 @@ protected EndPoint buildNodeEndPoint( // Don't rely on system.local.rpc_address for the control node, because it mistakenly // reports the normal RPC address instead of the broadcast one (CASSANDRA-11181). We // already know the endpoint anyway since we've just used it to query. + return connectedNodeEndPoint(localEndPoint); + } + } + + /** + * The endpoint to register the connected node under: the address the control channel actually + * reached, rather than the contact point it was reached through. + * + *

They differ when the control connection came up through a contact point, because a contact + * point is kept unresolved and {@code ChannelFactory} binds a {@linkplain PinnableEndPoint + * pinned} copy of it to the one address the channel reached -- a copy that, by that interface's + * contract, is identified exactly like the unpinned original. Registering the node under it would + * give it a hostname identity: metric names and tags derived from a name that denotes the + * whole cluster rather than this node. That is bad enough on its own, but the real damage is that + * the identity is not the node's: the reconnection fallback hands the contact points back on + * every reconnection round, so each successive control node acquires the same one. Two live nodes + * then report under a single metric prefix, sharing get-or-create metric objects, until the next + * refresh moves the older one back to its own address -- and {@code clearMetrics()} recomputes + * the names to delete from the prefix the node still holds, taking the newcomer's freshly + * registered series with it (see {@code DefaultNode#setEndPoint}). + * + *

Deriving the identity from the address actually connected to fixes all of that at once: it + * is this node's own address, so it is unique to it, and re-registering an unchanged control node + * becomes a no-op instead of an identity change. + * + *

Endpoints this cannot rebuild are returned untouched -- a third-party {@link EndPoint}, or + * one whose {@code resolve()} is not a resolved {@code InetSocketAddress} (the user disabled + * Netty's resolver, or a custom one declined the address, so nothing was pinned). So is one that + * already carries the connected address, which is every reconnection to an identified node: the + * existing instance is kept so that the control node's endpoint stays {@code ==} to the + * channel's, which is what keeps {@link #refreshNode}'s control-node check an identity + * comparison. + */ + private static EndPoint connectedNodeEndPoint(EndPoint localEndPoint) { + if (!(localEndPoint instanceof DefaultEndPoint)) { + return localEndPoint; + } + SocketAddress connected = localEndPoint.resolve(); + if (!(connected instanceof InetSocketAddress) + || ((InetSocketAddress) connected).isUnresolved()) { return localEndPoint; } + DefaultEndPoint asConnected = new DefaultEndPoint((InetSocketAddress) connected); + return asConnected.asMetricPrefix().equals(localEndPoint.asMetricPrefix()) + ? localEndPoint + : asConnected; } // Called when a new node is being added; the peers table is keyed by broadcast_address, diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java new file mode 100644 index 00000000000..383eda9219d --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java @@ -0,0 +1,549 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThat; +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.connection.ConnectionInitException; +import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; +import com.datastax.oss.protocol.internal.Frame; +import com.datastax.oss.protocol.internal.ProtocolConstants; +import com.datastax.oss.protocol.internal.request.Options; +import com.datastax.oss.protocol.internal.request.Register; +import com.datastax.oss.protocol.internal.request.Startup; +import com.datastax.oss.protocol.internal.response.Error; +import com.datastax.oss.protocol.internal.response.Ready; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +/** + * Verifies the two steps {@link ChannelFactory} runs between protocol initialization and the + * completion of a candidate attempt: the caller's {@link ConnectHook}, and the REGISTER request + * that moved out of the init handshake so that a channel the hook is about to reject never + * registers for events. + */ +public class ChannelFactoryConnectHookTest extends ChannelFactoryTestBase { + + private static final Duration HOOK_TIMEOUT = Duration.ofSeconds(5); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + private void givenNegotiableProtocol() { + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + } + + /** + * Drives one candidate's handshake to successful completion. Only the factory's first channel + * sends OPTIONS, so later candidates start straight at STARTUP. + */ + private void completeInit() { + Frame requestFrame = readOutboundFrame(); + if (requestFrame.message instanceof Options) { + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + } + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Ready()); + writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("mockClusterName")); + } + + private static DriverChannelOptions optionsWithHook(ConnectHook hook) { + return DriverChannelOptions.builder().withConnectHook(hook, HOOK_TIMEOUT).build(); + } + + private static DriverChannelOptions optionsWithHookAndEvents(ConnectHook hook) { + return DriverChannelOptions.builder() + .withConnectHook(hook, HOOK_TIMEOUT) + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + } + + @Test + public void should_complete_candidate_only_after_hook_accepts() { + // Given — a hook whose completion the test controls. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + List vettedChannels = new CopyOnWriteArrayList<>(); + ConnectHook hook = + channel -> { + vettedChannels.add(channel); + return gate; + }; + + // When — init completes but the hook has not answered yet. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the attempt is not successful until the hook says so. (The hook runs on the event + // loop after init succeeds, so wait for the invocation before asserting on the future.) + await().atMost(java.time.Duration.ofSeconds(2)).until(() -> vettedChannels.size() == 1); + assertThat(channelFuture.toCompletableFuture()).isNotDone(); + + // When + gate.complete(null); + + // Then — the vetted channel is the one handed to the caller. + assertThatStage(channelFuture) + .isSuccess(channel -> assertThat(channel).isSameAs(vettedChannels.get(0))); + } + + @Test + public void should_try_next_address_when_hook_rejects_candidate() { + // Given — a name expanding to two addresses of the live server, and a hook that rejects the + // first candidate and accepts the second: the caller's acceptance criteria are per address, + // and a rejection must not write off the endpoint. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + AtomicInteger invocations = new AtomicInteger(); + ConnectHook hook = + channel -> { + CompletableFuture result = new CompletableFuture<>(); + if (invocations.incrementAndGet() == 1) { + result.completeExceptionally(new IllegalStateException("not this one")); + } else { + result.complete(null); + } + return result; + }; + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + optionsWithHook(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + completeInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + assertThat(invocations.get()).isEqualTo(2); + } + + @Test + public void should_fail_connect_when_hook_rejects_the_last_candidate() { + // Given — a single address, so the rejection has nowhere to advance to. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + IllegalStateException rejection = new IllegalStateException("cannot identify itself"); + ConnectHook hook = + channel -> { + CompletableFuture result = new CompletableFuture<>(); + result.completeExceptionally(rejection); + return result; + }; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the rejection's cause is preserved for diagnosis. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).isSameAs(rejection); + }); + } + + @Test + public void should_treat_synchronous_hook_throw_as_rejection() { + // A hook is a caller-supplied callback running inside a Netty listener: a leaked throwable + // would otherwise leave the attempt hanging forever. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + IllegalStateException thrown = new IllegalStateException("hook blew up"); + ConnectHook hook = + channel -> { + throw thrown; + }; + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).isSameAs(thrown); + }); + } + + @Test + public void should_reject_candidate_when_hook_times_out() { + // Given — a hook whose stage never completes; only the driver can bound that. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + ConnectHook hook = channel -> new CompletableFuture<>(); + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ofMillis(100)).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("timed out")); + } + + @Test + public void should_register_for_events_only_after_hook_accepts() { + // Given — events requested and a gated hook. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + ConnectHook hook = channel -> gate; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHookAndEvents(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + // The hook has not accepted yet: were REGISTER part of init, it would already be on the wire. + assertThat(tryReadOutboundFrame(200)).isNull(); + gate.complete(null); + + // Then — REGISTER goes out only now, and the attempt completes once it is acknowledged. + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + assertThat(((Register) registerFrame.message).eventTypes).containsExactly("foo", "bar"); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_fail_candidate_when_the_step_after_the_hook_throws() + throws InterruptedException { + // Given — events requested, and a config read that starts throwing once the hook has accepted + // (an option gone missing across a config reload, a custom DriverConfig, a wrong type). + // registerForEvents() runs inside the hook stage's whenComplete callback: nobody consumes the + // stage that callback returns, and the hook timeout that would have failed the candidate has + // just been cancelled, so without a blanket catch there the attempt would hang forever. + givenNegotiableProtocol(); + AtomicBoolean poisoned = new AtomicBoolean(); + when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT)) + .thenAnswer( + invocation -> { + if (poisoned.get()) { + throw new IllegalStateException("config reloaded without the option"); + } + return Duration.ofMillis(500); + }); + ChannelFactory factory = newChannelFactory(); + AtomicReference vettedChannel = new AtomicReference<>(); + ConnectHook hook = + channel -> { + vettedChannel.set(channel); + poisoned.set(true); + return CompletableFuture.completedFuture(null); + }; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHookAndEvents(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the attempt fails instead of hanging, REGISTER never goes out, and the channel it had + // already opened is closed rather than left dangling with nothing holding it. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).hasMessageContaining("config reloaded"); + }); + assertThat(tryReadOutboundFrame(200)).isNull(); + assertThat(vettedChannel.get().closeFuture().await(500, TimeUnit.MILLISECONDS)) + .as("the abandoned candidate's channel should have been closed") + .isTrue(); + } + + @Test + public void should_register_for_events_after_init_when_no_hook_is_set() { + // Given — events but no hook (REGISTER still has to happen even when there is nothing to vet). + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_try_next_address_when_registration_fails() { + // Given — two addresses; the first candidate's REGISTER is refused. A registration failure is + // a per-candidate failure, exactly as it was when REGISTER was an init step. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Error(ProtocolConstants.ErrorCode.SERVER_ERROR, "nope")); + + // Then — the loop advances; the second candidate registers successfully. + completeInit(); + registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_treat_a_zero_hook_timeout_as_unbounded() { + // The hook timeout comes from advanced.control-connection.timeout, and every other consumer of + // a + // driver timeout option reads a non-positive duration as "no timeout" (see + // AdminRequestHandler#onWriteComplete). Scheduled anyway, a zero delay fires on the next + // event-loop turn -- before any round trip can complete -- and would abandon every candidate of + // every contact point, so an operator who disabled that timeout could not initialize a session. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + ConnectHook hook = channel -> gate; + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ZERO).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the hook is given as long as it needs, and the candidate survives. + assertThat(tryReadOutboundFrame(200)).isNull(); + assertThat(channelFuture.toCompletableFuture()).isNotDone(); + gate.complete(null); + + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_not_latch_negotiated_state_from_a_candidate_the_hook_rejects() { + // The protocol version and cluster name used to be latched as soon as the transport connect and + // init handshake succeeded, which was safe while init had the last word on a candidate. It no + // longer does: this hook rejects one, and REGISTER (below) can too. A stale DNS record pointing + // at a foreign cluster would otherwise leave that cluster's name latched here, and every later + // connection -- to any node -- would fail its cluster-name check, which ChannelPool turns into + // an irreversible forced-down node. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + ConnectHook hook = + channel -> { + CompletableFuture rejected = new CompletableFuture<>(); + rejected.completeExceptionally(new IllegalStateException("no host_id")); + return rejected; + }; + + // When — the only candidate is rejected after a fully successful handshake. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("hook")); + assertThat(factory.protocolVersion).isNull(); + assertThat(factory.getClusterName()).isNull(); + } + + @Test + public void should_not_latch_negotiated_state_from_a_candidate_whose_registration_fails() { + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Error(ProtocolConstants.ErrorCode.SERVER_ERROR, "nope")); + + // Then + assertThatStage(channelFuture).isFailed(); + assertThat(factory.protocolVersion).isNull(); + assertThat(factory.getClusterName()).isNull(); + } + + @Test + public void should_latch_negotiated_state_once_a_candidate_is_accepted() { + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHook(channel -> CompletableFuture.completedFuture(null)), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + assertThatStage(channelFuture).isSuccess(); + assertThat(factory.protocolVersion).isEqualTo(DefaultProtocolVersion.V4); + assertThat(factory.getClusterName()).isEqualTo("mockClusterName"); + } + + @Test + public void should_stop_the_candidate_loop_when_the_server_does_not_know_the_event_type() { + // Given — two addresses, and a server that does not support the event type being registered + // for. Unlike every other REGISTER failure this one is node-wide: it describes the software + // running there, not the record that reached it, so replaying it against the remaining + // addresses can only fail the same way while paying a full TCP connect plus the + // STARTUP/AUTH/cluster-name handshake for each. Stopping at the first restores what this + // rejection cost while REGISTER was an init step, which was one failed connect per contact + // point. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + // Then — the attempt is already failed, without the second address having been dialled. Compare + // should_try_next_address_when_registration_fails, where the stage is still pending at this + // point because the loop moved on. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error).hasMessageContaining("CLIENT_ROUTES_CHANGE"); + }); + } + + @Test + public void should_translate_client_routes_register_rejection() { + // The one REGISTER rejection with a known cause keeps its clear message, as it had when + // REGISTER was an init step: the caller (ClientRoutesTopologyMonitor.init()) reports it + // instead of silently degrading. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error).hasMessageContaining("CLIENT_ROUTES_CHANGE"); + assertThat(error).hasMessageContaining("ScyllaDB Enterprise >= 2026.1"); + }); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java index 682caac198d..322af30dc84 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java @@ -36,7 +36,6 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; -import com.datastax.oss.driver.api.core.connection.ConnectionInitException; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.internal.core.DefaultProtocolVersionRegistry; import com.datastax.oss.driver.internal.core.ProtocolVersionRegistry; @@ -50,11 +49,9 @@ import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.ProtocolConstants; -import com.datastax.oss.protocol.internal.ProtocolConstants.ErrorCode; import com.datastax.oss.protocol.internal.request.AuthResponse; import com.datastax.oss.protocol.internal.request.Options; import com.datastax.oss.protocol.internal.request.Query; -import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.AuthChallenge; import com.datastax.oss.protocol.internal.response.AuthSuccess; @@ -159,6 +156,52 @@ public void should_initialize() { assertThat(connectFuture).isSuccess(); } + /** + * Drives initialization as far as the cluster name check, which is where node identification + * branches off. + * + * @return the connect future, so that the caller can assert on its outcome. + */ + private ChannelFuture initUpToClusterName(DriverChannelOptions options) { + channel + .pipeline() + .addLast( + ChannelFactory.INIT_HANDLER_NAME, + new ProtocolInitHandler( + internalDriverContext, + DefaultProtocolVersion.V4, + null, + END_POINT, + options, + heartbeatHandler, + false)); + + ChannelFuture connectFuture = channel.connect(new InetSocketAddress("localhost", 9042)); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(buildInboundFrame(requestFrame, new Ready())); + + requestFrame = readOutboundFrame(); + assertThat(((Query) requestFrame.message).query) + .isEqualTo("SELECT cluster_name FROM system.local WHERE key='local'"); + writeInboundFrame(requestFrame, TestResponses.clusterNameResponse("someClusterName")); + + return connectFuture; + } + + @Test + public void should_complete_init_after_cluster_name_check() { + // Given — no keyspace to set. The node-identity read and the REGISTER request both happen + // after initialization (through the connect hook and ChannelFactory respectively), so init + // itself ends at the cluster-name check: byte for byte the pre-multi-address exchange. + ChannelFuture connectFuture = initUpToClusterName(DriverChannelOptions.DEFAULT); + + // Then + assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); + } + // Mirrors the real reporter, which only ever sees the control connection. private void stubConfigReporter() { when(internalDriverContext.getDriverConfigReporter()) @@ -601,7 +644,10 @@ public void should_initialize_with_keyspace() { } @Test - public void should_initialize_with_events() { + public void should_not_send_register_during_init_even_when_events_are_requested() { + // REGISTER moved out of initialization: ChannelFactory sends it after the connect hook has + // accepted the channel, so a candidate about to be rejected never registers for events. Init + // itself must therefore end without a Register frame even when the options ask for events. List eventTypes = ImmutableList.of("foo", "bar"); EventCallback eventCallback = mock(EventCallback.class); DriverChannelOptions driverChannelOptions = @@ -624,12 +670,8 @@ public void should_initialize_with_events() { writeInboundFrame(readOutboundFrame(), new Ready()); writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("someClusterName")); - Frame requestFrame = readOutboundFrame(); - assertThat(requestFrame.message).isInstanceOf(Register.class); - assertThat(((Register) requestFrame.message).eventTypes).containsExactly("foo", "bar"); - writeInboundFrame(requestFrame, new Ready()); - assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); } @Test @@ -664,12 +706,10 @@ public void should_initialize_with_keyspace_and_events() { assertThat(((Query) requestFrame.message).query).isEqualTo("USE \"ks\""); writeInboundFrame(requestFrame, new SetKeyspace("ks")); - requestFrame = readOutboundFrame(); - assertThat(requestFrame.message).isInstanceOf(Register.class); - assertThat(((Register) requestFrame.message).eventTypes).containsExactly("foo", "bar"); - writeInboundFrame(requestFrame, new Ready()); - + // No Register frame: setting the keyspace is now the last init step, events are registered by + // ChannelFactory after the connect hook. assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); } @Test @@ -745,56 +785,4 @@ public void should_fail_pending_requests_only_once_if_init_fails() { logger.detachAppender(appender); logger.setLevel(levelBefore); } - - @Test - public void should_fail_connection_when_client_routes_change_rejected() { - List eventTypes = - ImmutableList.of( - ProtocolConstants.EventType.SCHEMA_CHANGE, - ProtocolConstants.EventType.STATUS_CHANGE, - ProtocolConstants.EventType.TOPOLOGY_CHANGE, - ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); - EventCallback eventCallback = mock(EventCallback.class); - DriverChannelOptions driverChannelOptions = - DriverChannelOptions.builder().withEvents(eventTypes, eventCallback).build(); - channel - .pipeline() - .addLast( - ChannelFactory.INIT_HANDLER_NAME, - new ProtocolInitHandler( - internalDriverContext, - DefaultProtocolVersion.V4, - null, - END_POINT, - driverChannelOptions, - heartbeatHandler, - false)); - - ChannelFuture connectFuture = channel.connect(new InetSocketAddress("localhost", 9042)); - - // STARTUP - writeInboundFrame(readOutboundFrame(), new Ready()); - // Cluster name check - writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("someClusterName")); - - // REGISTER attempt includes CLIENT_ROUTES_CHANGE - Frame registerFrame = readOutboundFrame(); - assertThat(registerFrame.message).isInstanceOf(Register.class); - Register firstRegister = (Register) registerFrame.message; - assertThat(firstRegister.eventTypes).contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); - - // Server rejects with PROTOCOL_ERROR mentioning CLIENT_ROUTES_CHANGE - writeInboundFrame( - registerFrame, - new Error( - ErrorCode.PROTOCOL_ERROR, - "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); - - // Connection must fail with a clear error message - assertThat(connectFuture).isFailed(); - assertThat(connectFuture.cause()) - .isInstanceOf(ConnectionInitException.class) - .hasMessageContaining("CLIENT_ROUTES_CHANGE") - .hasMessageContaining("ScyllaDB Enterprise >= 2026.1"); - } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java index 3b7c47535ea..2189d9a7847 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java @@ -21,35 +21,45 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.AllNodesFailedException; import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; +import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; import com.datastax.oss.driver.internal.core.channel.ChannelEvent; import com.datastax.oss.driver.internal.core.channel.DriverChannel; +import com.datastax.oss.driver.internal.core.channel.DriverChannelOptions; import com.datastax.oss.driver.internal.core.channel.MockChannelFactoryHelper; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.internal.core.metadata.DefaultNodeInfo; import com.datastax.oss.driver.internal.core.metadata.DistanceEvent; import com.datastax.oss.driver.internal.core.metadata.NodeInfo; import com.datastax.oss.driver.internal.core.metadata.NodeStateEvent; +import com.datastax.oss.driver.internal.core.metadata.SniEndPoint; import com.datastax.oss.driver.internal.core.metadata.TestNodeFactory; import com.datastax.oss.driver.internal.core.metadata.TopologyMonitor; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import java.net.ConnectException; +import java.net.InetSocketAddress; import java.time.Duration; import java.util.AbstractMap.SimpleEntry; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -66,6 +76,115 @@ public void should_close_successfully_if_it_was_never_init() { assertThatStage(closeFuture).isSuccess(); } + @Test + public void should_arm_connect_hook_for_a_node_whose_host_id_is_unknown() { + // Given — a contact point: DefaultNode.newContactPoint leaves hostId null, and that is the one + // case with something to learn. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + + // When + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // Then — the identity read happens through the connect hook, inside the factory's candidate + // loop, where an address that cannot identify itself is rejected while the hostname's other + // addresses are still on hand. + DriverChannelOptions options = connectOptions(contactPoint); + assertThat(options.connectHook).isNotNull(); + assertThat(options.connectHookTimeout).isNotNull(); + } + + @Test + public void should_reject_candidate_when_hook_reads_no_host_id() { + // Given — the hook armed for a contact point, and a topology monitor that answers the identity + // read with a NodeInfo carrying no host id (a bootstrapping node). + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + NodeInfo anonymousInfo = mock(NodeInfo.class); + when(anonymousInfo.getHostId()).thenReturn(null); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(anonymousInfo)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When — the factory invokes the hook against the candidate channel. + CompletionStage vetted = connectOptions(contactPoint).connectHook.onConnect(channel); + + // Then — the candidate is rejected, so the factory can move on to the next address; accepting + // it would fail later in registerNode and cost the whole plan entry. + assertThatStage(vetted).isFailed(error -> assertThat(error).hasMessageContaining("host id")); + } + + @Test + public void should_not_arm_connect_hook_for_a_node_that_is_already_identified() { + // Given — node1 has a host id, so there is nothing to learn and nothing to vet: its + // connections keep the exact exchange they had before. + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + + // When + controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + + // Then + assertThat(connectOptions(node1).connectHook).isNull(); + } + + @Test + public void should_use_node_info_captured_by_hook_without_reading_again() { + // Given — a contact point whose connect hook has run against the winning channel, capturing + // the identity it read. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + CompletableFuture channelFuture = new CompletableFuture<>(); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .pending(contactPoint, channelFuture) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + EndPoint channelEndPoint = channel.getEndPoint(); + NodeInfo nodeInfo = mock(NodeInfo.class); + when(nodeInfo.getHostId()).thenReturn(UUID.randomUUID()); + when(nodeInfo.getEndPoint()).thenReturn(channelEndPoint); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(nodeInfo)); + DefaultNode registered = TestNodeFactory.newNode(3, context); + when(metadataManager.registerNode(nodeInfo)) + .thenReturn(CompletableFuture.completedFuture(registered)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When — the hook runs once (as the factory would against the accepted candidate), then the + // connect completes with that same channel. + assertThatStage(connectOptions(contactPoint).connectHook.onConnect(channel)).isSuccess(); + channelFuture.complete(channel); + + // Then — what the hook captured is what gets registered, with no second identity read: the + // monitor was asked exactly once, by the hook itself. + verify(metadataManager, VERIFY_TIMEOUT).registerNode(nodeInfo); + verify(topologyMonitor, times(1)).getChannelNodeInfo(channel); + } + + /** The options {@code ChannelFactory} was asked to connect {@code node} with. */ + private DriverChannelOptions connectOptions(Node node) { + ArgumentCaptor captor = + ArgumentCaptor.forClass(DriverChannelOptions.class); + verify(channelFactory, VERIFY_TIMEOUT).connect(eq(node), captor.capture()); + return captor.getValue(); + } + @Test public void should_init_with_first_contact_point_if_reachable() { // Given @@ -1008,6 +1127,67 @@ public void should_not_report_auth_failure_when_a_node_failed_on_something_else( assertThat(ControlConnection.isAuthFailure(error)).isFalse(); } + @Test + public void should_try_next_node_if_the_node_behind_a_contact_point_is_excluded() { + // Given — init on node1, then a reconnection through a contact point, the way the fallback + // plan appends the original contact points to every reconnection round. + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel3 = newMockDriverChannel(3); + + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + UUID resolvedHostId = UUID.randomUUID(); + DefaultNode resolvedNode = TestNodeFactory.newNode(2, resolvedHostId, context); + + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) + .success(contactPoint, channel2) + .success(node1, channel3) + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture) + .isSuccess(v -> assertThat(controlConnection.channel()).isEqualTo(channel1)); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(node1)); + + NodeInfo resolvedInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel2.getEndPoint()) + .withHostId(resolvedHostId) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())) + .thenAnswer( + invocation -> { + registeredNodes.put(resolvedNode.getHostId(), resolvedNode); + return CompletableFuture.completedFuture(resolvedNode); + }); + + // The backend that contact point reaches is one the policy has excluded. The event names the + // metadata node, because that is the only instance events ever name: the contact point the + // plan offered is an ephemeral object that is never their subject, so the guards that run + // before the handshake have nothing to match it against. + eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, resolvedNode)); + + // When + mockQueryPlan(contactPoint, node1); + channel1.close(); + + // Then — the channel is abandoned once the handshake says which node is behind it, and the + // next node is tried. Parking here instead would be permanent: an unchanged distance fires no + // further event, so nothing downstream would ever dislodge it. + verify(reconnectionSchedule, VERIFY_TIMEOUT).nextDelay(); + factoryHelper.waitForCall(contactPoint); + verify(channel2, VERIFY_TIMEOUT).forceClose(); + verify(eventBus, never()).fire(ChannelEvent.channelOpened(resolvedNode)); + factoryHelper.waitForCall(node1); + } + @Test public void should_not_report_auth_failure_for_a_throwable_with_no_per_node_breakdown() { // Only an AllNodesFailedException carries the per-node errors this question is answered from. @@ -1017,6 +1197,199 @@ public void should_not_report_auth_failure_for_a_throwable_with_no_per_node_brea .isFalse(); } + @Test + public void should_treat_a_lone_authentication_failure_as_auth_only() { + assertThat(ControlConnection.isAuthOnly(authError(node1))).isTrue(); + } + + @Test + public void + should_not_treat_an_authentication_failure_with_other_suppressed_causes_as_auth_only() { + // What the per-node log line is decided on. ChannelFactory#surfacedFailure deliberately + // promotes + // an authentication failure over transport ones, so the top-level throwable of a hostname whose + // records failed [refused, refused, auth] is the auth error -- and reporting that as "an + // authentication error" would say nothing about two thirds of the deployment being unreachable. + Throwable withUnreachableSibling = authError(node1); + withUnreachableSibling.addSuppressed(new ConnectException("connection refused")); + + assertThat(ControlConnection.isAuthOnly(withUnreachableSibling)).isFalse(); + } + + @Test + public void should_not_treat_a_non_authentication_failure_as_auth_only() { + assertThat(ControlConnection.isAuthOnly(new ConnectException("connection refused"))).isFalse(); + } + + @Test + public void should_never_pair_captured_node_info_with_a_channel_it_did_not_come_from() + throws Exception { + // The candidate loop can leave a stranded hook behind: an attempt abandoned on the hook timeout + // whose system.local response arrives anyway, and writes to the holder after the accepted + // candidate has. Held in two separate volatile fields, that late write lands between the + // accepted candidate's two writes, and a reader in that window takes the rejected candidate's + // node info as the accepted channel's -- registering the wrong host id and endpoint for the + // node + // the control connection is actually talking to. Published as one reference, the worst a late + // write can do is make the read miss and fall back to querying the open channel again. + ControlConnection.NodeInfoHolder holder = new ControlConnection.NodeInfoHolder(); + DriverChannel accepted = newMockDriverChannel(1); + DriverChannel abandoned = newMockDriverChannel(2); + NodeInfo acceptedInfo = mock(NodeInfo.class); + NodeInfo abandonedInfo = mock(NodeInfo.class); + AtomicReference mismatch = new AtomicReference<>(); + AtomicBoolean stop = new AtomicBoolean(); + + Thread writer = + new Thread( + () -> { + while (!stop.get()) { + holder.set(abandoned, abandonedInfo); + holder.set(accepted, acceptedInfo); + } + }); + writer.start(); + try { + for (int i = 0; i < 200_000 && mismatch.get() == null; i++) { + NodeInfo forAccepted = holder.getFor(accepted); + if (forAccepted != null && forAccepted != acceptedInfo) { + mismatch.set(forAccepted); + } + NodeInfo forAbandoned = holder.getFor(abandoned); + if (forAbandoned != null && forAbandoned != abandonedInfo) { + mismatch.set(forAbandoned); + } + } + } finally { + stop.set(true); + writer.join(); + } + + assertThat(mismatch.get()).isNull(); + } + + @Test + public void should_report_why_it_gave_up_when_the_node_behind_a_contact_point_is_excluded() { + // Given -- a plan with nothing but a contact point whose backend turns out to be excluded. The + // event names the metadata node, because that is the only instance events ever name. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(2); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + UUID resolvedHostId = UUID.randomUUID(); + DefaultNode resolvedNode = TestNodeFactory.newNode(2, resolvedHostId, context); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + // Built outside the when(): calling a mock inside an unfinished stubbing confuses Mockito. + NodeInfo resolvedInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel.getEndPoint()) + .withHostId(resolvedHostId) + .build(); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())) + .thenReturn(CompletableFuture.completedFuture(resolvedNode)); + eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, resolvedNode)); + + // When -- init, with no other node to fall back to. + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // Then -- the exclusion is recorded as this node's error, so the operator is told why rather + // than getting a bare NoNodeAvailableException with no cause at all. + assertThatStage(initFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(AllNodesFailedException.class); + assertThat(((AllNodesFailedException) error).getAllErrors()) + .hasEntrySatisfying( + resolvedNode, + errors -> assertThat(errors.get(0)).hasMessageContaining("ignored")); + }); + } + + @Test + public void should_reconnect_when_the_node_a_pending_channel_reached_becomes_excluded() { + // Given -- a contact point that is a hostname, so the channel's endpoint (bound by + // ChannelFactory to the address it actually reached) is the only thing that says which node is + // at the other end while the identity read is still in flight. + DefaultNode contactPoint = + DefaultNode.newContactPoint( + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.invalid", 9042)), + context); + mockQueryPlan(contactPoint); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(contactPoint, channel2) + .success(node1, channel1) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)).thenReturn(new CompletableFuture<>()); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- a metadata node at the address that channel is connected to is forced down, while the + // identity read has not come back yet. + mockQueryPlan(node1); + eventBus.fire( + NodeStateEvent.changed( + NodeState.UP, NodeState.FORCED_DOWN, TestNodeFactory.newNode(2, context))); + + // Then -- the control connection recognizes that as the node it is sitting on and moves. It + // cannot compare host ids yet, and comparing the two endpoints with equals() would resolve the + // contact-point hostname inline -- a blocking DNS lookup on the admin thread, answered by + // whichever address the name lists first, which need not even be this node. + factoryHelper.waitForCall(node1); + } + + @Test + public void should_reconnect_when_a_pending_proxy_channel_reached_an_excluded_node() { + // Given -- a Cloud-style deployment. The contact point and the metadata node are SniEndPoints + // over the same proxy and server name, and the channel carries the copy ChannelFactory pinned + // to + // the proxy IP it actually reached. resolve() therefore answers an unresolved proxy name on the + // node side and a concrete IP on the channel side, and an InetSocketAddress carrying an + // InetAddress never equals one that does not -- so comparing resolve() results here would say + // "not the control node" for every Cloud, client-route and unresolved-translator deployment. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.invalid", 9142); + String serverName = "1e9a4d0c-0000-0000-0000-00000000002a"; + SniEndPoint proxyEndPoint = new SniEndPoint(proxy, serverName); + DefaultNode contactPoint = DefaultNode.newContactPoint(proxyEndPoint, context); + mockQueryPlan(contactPoint); + DriverChannel channel2 = newMockDriverChannel(2); + when(channel2.getEndPoint()) + .thenReturn(proxyEndPoint.pinTo(new InetSocketAddress("127.0.0.2", 9142))); + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(contactPoint, channel2) + .success(node1, channel1) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)).thenReturn(new CompletableFuture<>()); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- the metadata node behind that same proxy is forced down, while the identity read has + // not come back yet. + mockQueryPlan(node1); + eventBus.fire( + NodeStateEvent.changed( + NodeState.UP, + NodeState.FORCED_DOWN, + new DefaultNode(new SniEndPoint(proxy, serverName), context))); + + // Then -- the control connection recognizes that as the node it is sitting on and moves. The + // proxy endpoint's identity does not key on the address at all (proxy + server name), so + // equals() answers this without resolving anything. + factoryHelper.waitForCall(node1); + } + private static Throwable authError(Node node) { return new AuthenticationException(node.getEndPoint(), "mock authentication failure"); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java index 892715a47c3..a0ef5197734 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java @@ -38,6 +38,7 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; import com.datastax.oss.driver.internal.core.addresstranslation.PassThroughAddressTranslator; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; @@ -443,6 +444,76 @@ public void should_fail_get_channel_node_info_if_local_result_is_empty() { }); } + @Test + public void should_query_system_local_for_channel_node_info() { + // Given — a channel to identify (the control connection's connect hook is the caller). + UUID hostId = UUID.randomUUID(); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", mockResult(mockLocalRow(1, hostId)))); + + // When + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + // Then + assertThatStage(futureNodeInfo) + .isSuccess(nodeInfo -> assertThat(nodeInfo.getHostId()).isEqualTo(hostId)); + } + + @Test + public void should_identify_the_connected_node_by_the_address_it_reached_not_the_contact_point() { + // Given — a control channel that came up through a contact-point hostname. ChannelFactory binds + // the endpoint it hands the channel to the one address that connection reached, and by + // PinnableEndPoint's contract that copy is identified exactly like the unpinned original. + EndPoint contactPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.com", 9042)); + EndPoint pinned = + ((PinnableEndPoint) contactPoint).pinTo(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(pinned.asMetricPrefix()).isEqualTo("db_example_com:9042"); + when(channel.getEndPoint()).thenReturn(pinned); + UUID hostId = UUID.randomUUID(); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", mockResult(mockLocalRow(1, hostId)))); + + // When + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + // Then — the node is registered under its own address, not under the name it was reached + // through. Registering it under the contact point would give it a hostname identity that is not + // even exclusively its own: the reconnection fallback hands the contact points back every + // round, + // so each successive control node would take the same metric prefix, and two live nodes sharing + // one prefix means the older one's clearMetrics() deletes the newcomer's series. + assertThatStage(futureNodeInfo) + .isSuccess( + nodeInfo -> { + assertThat(nodeInfo.getEndPoint().asMetricPrefix()).isEqualTo("127_0_0_1:9042"); + assertThat(nodeInfo.getEndPoint().resolve()) + .isEqualTo(new InetSocketAddress("127.0.0.1", 9042)); + }); + } + + @Test + public void should_keep_the_channels_own_endpoint_when_it_already_names_the_connected_address() { + // The reconnection-to-a-known-node case: nothing was pinned because the endpoint already held + // the address, so the existing instance is kept -- which is what keeps the control-node check + // in + // refreshNode() an identity comparison rather than an endpoint equals() (and DefaultEndPoint's + // equals resolves names). + EndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + when(channel.getEndPoint()).thenReturn(endPoint); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", + mockResult(mockLocalRow(1, UUID.randomUUID())))); + + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + assertThatStage(futureNodeInfo) + .isSuccess(nodeInfo -> assertThat(nodeInfo.getEndPoint()).isSameAs(endPoint)); + } + @Test public void should_stop_executing_queries_once_closed() { // Given From 68668239874908841e02185a56e1f2bd107c87a5 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:16:56 +0200 Subject: [PATCH 7/9] feat: fall back to the original contact points on control reconnection (DRIVER-201) advanced.control-connection.reconnection.fallback-to-original-contact-points now defaults to true, and is the driver's DNS re-resolution path. Nothing else re-resolves. Metadata nodes hold an endpoint built from an already-resolved system.peers IP, and the control node's own endpoint is pinned by ChannelFactory to the single address its connection reached, deliberately, so that a node with a known identity cannot wander to a different host. Once the records behind a hostname change, appending the original contact points is therefore the only way back: they are still unresolved hostnames, so ChannelFactory expands each one to its current IPs at connection time. The append is gated on the topology monitor not re-resolving addresses itself, since a proxy-based monitor keeps them fresh and raw contact points could resurrect nodes it has authoritatively removed. The exception is an empty regular plan: with no live node to try, reconnection cannot recover on its own. The plans are concatenated rather than mutated. A RUNNING-state query plan is a built-in QueryPlan whose add()/addAll() throw UnsupportedOperationException, poll() being its only mutator, so with the fallback defaulting on every post-init control reconnect would otherwise have crashed. The append is also skipped before the LBP reaches RUNNING, where newQueryPlan() has already built the plan from the contact points and appending would duplicate every entry. Documented cost: the contact points are appended without being compared against the live-node plan, because at plan time they are hostnames while the live nodes are resolved IPs. When DNS has not changed they expand to addresses the plan just failed on, so an exhausted reconnection round retries roughly twice as many addresses -- which is why HeartbeatIT has to disable it. Co-Authored-By: Claude Opus 5 (1M context) --- .../api/core/config/DefaultDriverOption.java | 10 +- .../driver/api/core/config/OptionsMap.java | 2 +- .../api/core/config/TypedDriverOption.java | 10 +- .../metadata/ClientRoutesTopologyMonitor.java | 18 ++- .../metadata/LoadBalancingPolicyWrapper.java | 77 +++++++-- .../core/metadata/MetadataManager.java | 6 + core/src/main/resources/reference.conf | 24 ++- .../ClientRoutesTopologyMonitorTest.java | 8 +- .../LoadBalancingPolicyWrapperTest.java | 148 ++++++++++++++---- .../driver/core/heartbeat/HeartbeatIT.java | 6 + 10 files changed, 255 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index f026558171c..da0a3c622cb 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -708,8 +708,14 @@ public enum DefaultDriverOption implements DriverOption { CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"), /** - * Whether to forcibly add original contact points held by MetadataManager to the reconnection - * plan, in case there is no live nodes available according to LBP. Experimental. + * Whether to append the original contact points held by MetadataManager to the reconnection plan, + * after the live nodes reported by the load balancing policy. Defaults to {@code true}. + * + *

This is also the driver's DNS re-resolution path. Contact points are appended as-is, still + * unresolved hostnames, and each is expanded to its current DNS IPs at connection time through + * Netty's configured resolver. Metadata nodes, in contrast, hold an already-resolved endpoint + * that is never re-resolved. Keeping this enabled lets control-connection reconnects re-resolve + * the original hostnames and pick up new IPs once the live-node plan is exhausted. * *

Value-type: boolean */ diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index 0d86d7b1c62..bc5bf51c2ba 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -373,7 +373,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true); - map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false); + map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true); map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true); map.put(TypedDriverOption.REPREPARE_ENABLED, true); map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index 8a8f5512644..fe7888682bc 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -607,7 +607,15 @@ public String toString() { public static final TypedDriverOption CONTROL_CONNECTION_AGREEMENT_WARN = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN); - /** Whether to forcibly try original contacts if no live nodes are available */ + /** + * Whether to append the original contact points to the control-connection reconnection plan, + * after the live nodes reported by the load balancing policy (defaults to {@code true}). + * + *

Contact points are appended as-is (unresolved hostnames); each is expanded to all of its + * current DNS IPs at connection time, which is also the driver's DNS re-resolution mechanism. The + * append is skipped for topology monitors that re-resolve node addresses themselves (such as the + * cloud/proxy monitors). + */ public static final TypedDriverOption CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java index 31d27aa7c7d..a189f16c565 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java @@ -35,6 +35,7 @@ import java.net.InetSocketAddress; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; @@ -497,8 +498,23 @@ public boolean reresolvesNodeAddresses() { // static, already-resolved fallback endpoint instead. Only report true when every // currently-known node actually has a live route; otherwise the contact-point reconnection // fallback must stay available for the nodes stuck on that fallback. + // + // "Every known node" has to mean at least one: with an empty node set the loop below would + // report true vacuously, suppressing the contact-point fallback at the one moment it is the + // only + // way back -- before the first node refresh, or after the monitor has removed everything. (The + // caller happens to exempt an empty query plan as well, but that is a separate safety net and + // this must not depend on it.) + // + // The answer legitimately changes as route coverage does, so successive reconnection rounds can + // see different values: that tracks reality rather than flapping. The scan is O(nodes) and runs + // once per reconnection attempt, against an in-memory map. + Collection nodes = context.getMetadataManager().getMetadata().getNodes().values(); + if (nodes.isEmpty()) { + return false; + } Map routes = resolvedRoutesCache.get(); - for (Node node : context.getMetadataManager().getMetadata().getNodes().values()) { + for (Node node : nodes) { if (!routes.containsKey(node.getHostId())) { return false; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index f3f3e4fe346..fd11f4c902f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -27,6 +27,8 @@ import com.datastax.oss.driver.api.core.session.Request; import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.util.collection.CompositeQueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.internal.core.util.concurrent.ReplayingEventFilter; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; @@ -147,7 +149,9 @@ public Queue newQueryPlan( switch (stateRef.get()) { case BEFORE_INIT: case DURING_INIT: - // The contact points are not stored in the metadata yet: + // The contact points are not stored in the metadata yet. Each unresolved hostname is + // expanded to all its DNS IPs at connection time by ChannelFactory, so one entry per + // contact point is enough here. List nodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(nodes); return new ConcurrentLinkedQueue<>(nodes); @@ -164,20 +168,69 @@ public Queue newQueryPlan( @NonNull public Queue newControlReconnectionQueryPlan() { + // Read the state once, before building the regular plan. State transitions are monotonic + // (BEFORE_INIT -> DURING_INIT -> RUNNING -> ...), so this captured value is <= the value + // newQueryPlan() reads internally; that guarantees we never both build the plan from the + // contact points (pre-RUNNING branch of newQueryPlan) and append them again below. + // + // Note: this is still two separate reads of stateRef (this one, and newQueryPlan()'s own + // internal read a moment later), so a transition landing exactly between them is possible: if + // state flips BEFORE_INIT/DURING_INIT -> RUNNING in that window, newQueryPlan() takes the + // RUNNING branch (a real LBP-built plan) while the state captured here is still pre-RUNNING, + // so the contact-point fallback below is skipped for this one call even though + // regularQueryPlan didn't come from the contact-point branch. This is benign: no crash, no + // duplicate entries, and it self-corrects on the very next reconnection attempt. + // + // Monotonicity leaves the other direction open, and it is worth naming: a RUNNING -> CLOSING + // flip in that same window makes newQueryPlan() take its default branch and return an empty + // plan, which passes both the RUNNING check below and the empty-plan exemption from the + // re-resolving-monitor rule, so the plan handed back is the contact points alone. Also benign: + // ControlConnection abandons a reconnection attempt on closeWasCalled, and every node in that + // plan is one it already had. + State state = stateRef.get(); Queue regularQueryPlan = newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - if (context - .getConfig() - .getDefaultProfile() - .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) { - Set originalNodes = context.getMetadataManager().getContactPoints(); - List contactNodes = new ArrayList<>(); - for (DefaultNode node : originalNodes) { - contactNodes.add(DefaultNode.newContactPoint(node.getEndPoint(), context)); - } + // Only append the contact points as an explicit fallback once the LBP is RUNNING: before that + // (BEFORE_INIT/DURING_INIT), newQueryPlan() above already built regularQueryPlan directly from + // the contact points, so appending them again here would just duplicate every entry. + // + // Skipped when the topology monitor re-resolves node addresses on its own (e.g. proxy-based + // monitors such as client routes or the cloud SNI proxy): those keep addresses fresh without + // this fallback, and appending raw contact points could resurrect nodes the monitor has + // authoritatively removed. The exception is an empty regular plan: with no live node to try, + // reconnection cannot recover on its own, so the contact-point fallback is kept even for those + // monitors. + if (state == State.RUNNING + && context + .getConfig() + .getDefaultProfile() + .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS) + && (!context.getTopologyMonitor().reresolvesNodeAddresses() + || regularQueryPlan.isEmpty())) { + // Append the original (unresolved) contact points so every IP their hostname resolves to is + // tried as a fallback: ChannelFactory expands each one at connection time, instead of the + // driver being stuck with whatever single IP a metadata node happens to hold. + // + // The retained instances, not fresh copies. MetadataManager holds the contact-point nodes for + // the session's lifetime and the pre-RUNNING branch of newQueryPlan() already hands out these + // very objects, so minting a copy per plan would give each reconnection round a distinct node + // firing its own controlConnectionFailed event -- one set per round, for as long as + // reconnection lasts. Shuffling a fresh list leaves the retained set itself untouched. + // + // Metrics are not a reason either way, and are worth stating because it looks as though they + // should be: DefaultNode.newContactPoint installs NoopNodeMetricUpdater, so a contact-point + // node records nothing -- no bytes, no errors.connection.init, no errors.connection.auth -- + // and a fresh copy would be no worse. Now that this fallback is on by default, that blind + // spot covers every reconnect that reaches a contact point rather than only session init, so + // an operator watching errors.connection.auth will not see a contact point failing to + // authenticate. Giving these nodes real updaters would register metrics under names for + // ephemeral objects that are deliberately absent from metadata, so it is left as is. + List contactNodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(contactNodes); - // Append contact points to the end of the regular query plan so they serve as a fallback - regularQueryPlan.addAll(contactNodes); + // Concatenate rather than mutate: the RUNNING-state regularQueryPlan is a built-in QueryPlan + // whose add()/addAll() throw UnsupportedOperationException (poll() is its only mutator). + // CompositeQueryPlan drains the regular plan first, then the contact-point fallback. + return new CompositeQueryPlan(regularQueryPlan, new SimpleQueryPlan(contactNodes.toArray())); } return regularQueryPlan; diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java index cd765c818e6..d8671678306 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java @@ -188,6 +188,12 @@ public boolean wasImplicitContactPoint() { * they are never added to metadata and never exposed to user-facing APIs (events, {@link * com.datastax.oss.driver.api.core.metadata.Metadata#getNodes()}, or {@link * com.datastax.oss.driver.api.core.metadata.NodeStateListener} callbacks). + * + *

The metadata node stores {@code nodeInfo.getEndPoint()} as-is and never re-resolves it on + * its own. Re-resolving the original contact-point hostname to pick up current DNS only happens + * through the original-contact-point reconnection fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}), which re-enters + * the contact points and lets {@code ChannelFactory} expand each hostname at connection time. */ public CompletionStage registerNode(NodeInfo nodeInfo) { Preconditions.checkNotNull(nodeInfo.getHostId(), "Cannot register node without hostId"); diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 72a1a7b7a97..ac1e3f39450 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -2362,14 +2362,30 @@ datastax-java-driver { } reconnection { - # Whether to forcibly add original contact points held by MetadataManager to the reconnection plan, - # in case there is no live nodes available according to LBP. - # Experimental. + # Whether to append the original contact points held by MetadataManager to the reconnection + # plan, after the live nodes reported by the load balancing policy. + # + # This is also the driver's DNS re-resolution path. Contact points are kept as unresolved + # hostnames and expanded to their current DNS IPs at connection time, through Netty's + # configured resolver. Metadata nodes, in contrast, store an already-resolved endpoint that + # is never re-resolved, so once DNS records change they would otherwise become stale. Keeping + # this enabled lets control-connection reconnects re-resolve the original hostnames and pick up + # the new IPs once the live-node plan is exhausted. + # + # Note the cost: the contact points are appended without being compared against the live-node + # plan, because at plan time they are still hostnames and the live nodes are already-resolved + # IPs. When DNS has not changed they therefore expand to addresses the plan just failed on, so + # a reconnection round that exhausts the live nodes retries roughly twice as many addresses. + # Each of those costs a connect attempt -- up to `advanced.connection.connect-timeout` -- and, + # if the address accepts the connection but then stalls, the init handshake on top, whose + # steps each arm their own `advanced.connection.init-query-timeout`. Set this to false if you + # do not need DNS re-resolution of contact points and would rather keep reconnection rounds + # short. # # Required: yes # Modifiable at runtime: yes, the new value will be used for checks issued after the change. # Overridable in a profile: no - fallback-to-original-contact-points = false + fallback-to-original-contact-points = true } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java index e6ab0895034..17f9e7ef136 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java @@ -274,12 +274,16 @@ public void should_not_reresolve_when_a_known_node_has_no_client_route() { } @Test - public void should_reresolve_when_no_nodes_known_yet() { + public void should_not_reresolve_when_no_nodes_are_known_yet() { + // "Every known node has a route" is vacuously true of an empty set, and answering yes there + // suppresses the contact-point reconnection fallback at the one moment it is the only way back: + // before the first node refresh, or after this monitor has removed every node. There is nothing + // for the monitor to keep fresh, so it must not claim it is keeping anything fresh. when(context.getMetadataManager()).thenReturn(metadataManager); when(metadataManager.getMetadata()).thenReturn(metadata); when(metadata.getNodes()).thenReturn(Collections.emptyMap()); - assertThat(handler.reresolvesNodeAddresses()).isTrue(); + assertThat(handler.reresolvesNodeAddresses()).isFalse(); } // ---- Merge behavior tests ----------------------------------------------- diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java index 89b36b9ee09..af0d331c98c 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java @@ -33,17 +33,17 @@ import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy; import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy.DistanceReporter; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; -import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Metadata; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; import com.datastax.oss.driver.internal.core.context.EventBus; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; +import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; -import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.Map; import java.util.Objects; import java.util.Queue; @@ -79,6 +79,7 @@ public class LoadBalancingPolicyWrapperTest { private EventBus eventBus; @Mock private MetadataManager metadataManager; @Mock private Metadata metadata; + @Mock private TopologyMonitor topologyMonitor; @Mock protected MetricsFactory metricsFactory; @Captor private ArgumentCaptor> initNodesCaptor; @@ -102,13 +103,16 @@ public void setup() { when(metadata.getNodes()).thenReturn(allNodes); when(metadataManager.getContactPoints()).thenReturn(contactPoints); when(context.getMetadataManager()).thenReturn(metadataManager); + when(context.getTopologyMonitor()).thenReturn(topologyMonitor); when(context.getConfig()).thenReturn(config); when(config.getDefaultProfile()).thenReturn(defaultProfile); when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(false); - defaultPolicyQueryPlan = Lists.newLinkedList(ImmutableList.of(node3, node2, node1)); + // Use a real built-in QueryPlan (not a mutable LinkedList): its add()/addAll() throw, so the + // control-reconnection plan must compose rather than mutate it (see CompositeQueryPlan usage). + defaultPolicyQueryPlan = new SimpleQueryPlan(node3, node2, node1); when(policy1.newQueryPlan(null, null)).thenReturn(defaultPolicyQueryPlan); eventBus = spy(new EventBus("test")); @@ -130,26 +134,28 @@ public void setup() { @Test public void should_build_control_connection_query_plan_from_contact_points_before_init() { - // When + // When — before init, the control-reconnection plan is built straight from the contact points + // (bypassing the load balancing policies), so each hostname can be tried on the first connect. Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test public void should_build_query_plan_from_contact_points_before_init() { - // When + // When — before init, the query plan is built straight from the contact points (bypassing the + // load balancing policies) Queue queryPlan = wrapper.newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test @@ -204,17 +210,106 @@ public void should_fetch_control_connection_query_plan_from_policy_after_init() assertThat(queryPlan.poll()).isEqualTo(node3); assertThat(queryPlan.poll()).isEqualTo(node2); assertThat(queryPlan.poll()).isEqualTo(node1); - // Remaining nodes are contact points appended at the end. - // They are new DefaultNode instances created via newContactPoint, so compare by endpoint. - Set remainingEndpoints = new java.util.HashSet<>(); - for (Node n : queryPlan) { - remainingEndpoints.add(n.getEndPoint()); + // Remaining nodes are the original contact points appended at the end. DefaultNode does not + // override equals, so comparing against the retained instances is an identity check. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); + } + + @Test + public void should_reuse_the_retained_contact_point_nodes_rather_than_minting_copies() { + // Given — the flag now defaults to true, so this plan is built on every control-connection + // reconnection round for every user, not only for those who opted in. + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); + + // When — two rounds, as a reconnection sequence would produce. + Queue firstPlan = wrapper.newControlReconnectionQueryPlan(); + Queue secondPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — both plans hand out the very objects MetadataManager retains, not per-plan copies. + // A copy would be invisible to the metric updater registered for the real node and would fire + // its own controlConnectionFailed event, once per contact point per round, for as long as + // reconnection lasts. + for (Node node : firstPlan) { + assertThat(contactPoints).contains((DefaultNode) node); } - Set contactEndpoints = new java.util.HashSet<>(); - for (DefaultNode n : contactPoints) { - contactEndpoints.add(n.getEndPoint()); + for (Node node : secondPlan) { + assertThat(contactPoints).contains((DefaultNode) node); } - assertThat(remainingEndpoints).isEqualTo(contactEndpoints); + // And the retained set itself is untouched by the shuffle. + assertThat(contactPoints).containsExactlyInAnyOrder(node1, node2); + } + + @Test + public void should_not_duplicate_contact_points_before_init() { + // Given — the wrapper hasn't been init()-ed yet (state=BEFORE_INIT), so newQueryPlan() already + // builds the regular plan directly from the contact points. The reconnect-contact-points flag + // doesn't matter here: newControlReconnectionQueryPlan() short-circuits on state before even + // reading it, since appending contact points again pre-init would just duplicate every entry in + // the plan. + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the contact points are read only once (not once for the regular plan and again for a + // redundant "fallback" append), and the plan has no duplicate entries. + verify(metadataManager, times(1)).getContactPoints(); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_reconnect_contact_points_is_disabled() { + // Given — the flag defaults to false in the test setup (see @Before) + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Only the policy query plan is returned; no contact points are appended. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_topology_monitor_reresolves_addresses() { + // Given — the flag is enabled, but the topology monitor re-resolves node addresses on its own + // (e.g. a proxy-based monitor such as client routes or the cloud SNI proxy). + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Contact points must not be appended: the monitor keeps addresses fresh, and appending raw + // contact points could resurrect nodes it has authoritatively removed. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_append_contact_points_when_query_plan_empty_even_if_topology_monitor_reresolves() { + // Given — the flag is enabled and the topology monitor re-resolves node addresses on its own, + // but the live-node query plan is empty. With no node to try, reconnection can only recover + // through the contact-point fallback, so it must be appended despite the re-resolving monitor. + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the retained contact-point instances are appended, in some order. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); } @Test @@ -223,24 +318,15 @@ public void should_return_contact_points_when_query_plan_empty_and_flag_enabled( when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(true); wrapper.init(); - // Make the policy return an empty query plan - when(policy1.newQueryPlan(null, null)).thenReturn(Lists.newLinkedList(ImmutableList.of())); + // Make the policy return an empty query plan (QueryPlan.EMPTY, as the real policies do) + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); // When Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); // Then - // Should get the contact points (compare by endpoint since they are new instances) - assertThat(queryPlan.size()).isEqualTo(contactPoints.size()); - Set resultEndpoints = new java.util.HashSet<>(); - for (Node n : queryPlan) { - resultEndpoints.add(n.getEndPoint()); - } - Set contactEndpoints = new java.util.HashSet<>(); - for (DefaultNode n : contactPoints) { - contactEndpoints.add(n.getEndPoint()); - } - assertThat(resultEndpoints).isEqualTo(contactEndpoints); + // Should get the retained contact-point instances themselves. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); } @Test diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java index 26658bd76d1..dcede822fca 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java @@ -235,6 +235,12 @@ private CqlSession newSession(ProgrammaticDriverConfigLoaderBuilder loaderBuilde .withDuration(DefaultDriverOption.HEARTBEAT_TIMEOUT, Duration.ofMillis(500)) .withDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT, Duration.ofSeconds(2)) .withDuration(DefaultDriverOption.RECONNECTION_MAX_DELAY, Duration.ofSeconds(1)) + // These tests exercise heartbeat behavior only. The contact-point reconnection + // fallback appends the contact points to a reconnection plan that has run out of live + // nodes, so a round tries roughly twice as many addresses -- and every one of those + // attempts sends an OPTIONS as part of protocol init. countHeartbeats() filters on + // isOptionRequest() alone, so it counts those alongside real heartbeats. Disable it. + .withBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false) .build(); return SessionUtils.newSession(SIMULACRON_RULE, loader); } From c0d028f51e604043fec644e7d4fa68efaf57e20f Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:17:17 +0200 Subject: [PATCH 8/9] docs: document multi-address DNS resolution (DRIVER-201) Rewrites the address-resolution manual page around the connection layer doing the expansion, and adds an upgrade-guide section covering what changes for users: - there is no public API change, but EndPoint.resolve() may now return an unresolved address for Cloud/SNI and client-route nodes, so a caller doing ((InetSocketAddress) resolve()).getAddress().getHostAddress() gets a NPE where it previously worked; getHostString() is the safe read; - advanced.resolve-contact-points is deprecated and inert; - fallback-to-original-contact-points defaults to true, with its cost stated; - a contact point that none of its addresses can identify is now marked down before initialization completes, firing an event it did not fire before; - AllNodesFailedException reports one entry per contact point, with each address's failure attached as a suppressed exception; - the one-time TaggingMetricIdGenerator node-tag rename for hand-built Cloud proxy addresses; - the afterBootstrapInitialized() contract change; - two protected methods removed from internal classes that a subclass could have overridden. Co-Authored-By: Claude Opus 5 (1M context) --- manual/core/address_resolution/README.md | 20 ++- upgrade_guide/README.md | 157 +++++++++++++++++++++++ 2 files changed, 170 insertions(+), 7 deletions(-) diff --git a/manual/core/address_resolution/README.md b/manual/core/address_resolution/README.md index ae44feea3ea..c692aa63336 100644 --- a/manual/core/address_resolution/README.md +++ b/manual/core/address_resolution/README.md @@ -183,13 +183,19 @@ datastax-java-driver { #### DNS resolution -DNS is resolved at connection time (not at route discovery time). The driver delegates to -`InetAddress.getByName()`, which is a blocking call that uses the JVM's built-in DNS cache -(30 s default TTL in the JDK). Because this runs on Netty I/O threads, slow or unresponsive -DNS can block connection establishment and impact driver throughput. To mitigate this, configure -the JVM DNS cache TTL via the `networkaddress.cache.ttl` security property (e.g. in -`$JAVA_HOME/conf/security/java.security` or programmatically with -`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`). +DNS is resolved at connection time (not at route discovery time), and through the same mechanism as +every other address the driver connects to: the route's hostname is handed to the connection layer +unresolved, and Netty's configured `AddressResolverGroup` expands it. A custom resolver installed via +`NettyOptions.afterBootstrapInitialized()` therefore applies to client routes as well, and a hostname +that maps to several addresses has all of them tried in turn. + +With Netty's default (JDK) resolver the lookup is a blocking `InetAddress` call that uses the JVM's +built-in DNS cache (30 s default TTL in the JDK). It runs on a Netty I/O event loop — never on the +admin event loop that drives control-connection reconnects — so it delays the connection attempt +itself. It is therefore worth configuring the JVM DNS cache TTL via the `networkaddress.cache.ttl` +security property (e.g. in `$JAVA_HOME/conf/security/java.security` or programmatically with +`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`), or installing +`DnsAddressResolverGroup` for non-blocking resolution. - **Route-map refresh** — the driver re-queries `system.client_routes` and atomically swaps the in-memory route map in two situations: diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index 83963f94775..511f9dda2ca 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -61,6 +61,163 @@ Binary compatibility is unaffected — an already-compiled subclass keeps workin override either method, you have to widen your override to `public` in order to recompile: Java does not allow an override to reduce visibility. +#### Contact points are expanded to all their DNS addresses at connection time + +Contact points backed by a hostname are now kept unresolved and expanded to **all** the IP +addresses the hostname maps to, at connection time. Previously only the first address returned by +DNS was tried, so a single non-responsive IP behind a multi-record hostname could fail the initial +connection (or a control-connection reconnect) even when the other addresses were healthy. No +configuration change is required to benefit from this. + +The expansion goes through Netty's configured `AddressResolverGroup`, which is the resolver an +unresolved address already reached when it was handed to `Bootstrap.connect()`. A custom resolver +installed through `NettyOptions.afterBootstrapInitialized()` therefore keeps working; note that the +driver now calls `resolveAll()` on it, where previously `Bootstrap.connect()` called `resolve()`. +Every resolver built on Netty's own base classes implements both. As before, with Netty's default +(JDK) resolver the lookup blocks the I/O event loop it runs on — install `DnsAddressResolverGroup` +for non-blocking resolution. + +`Bootstrap.disableResolver()` is still honored, in the sense that the driver resolves nothing and +passes the address through as-is. That now requires the address to be usable as-is, which is a +narrower contract than before: contact-point hostnames are always kept unresolved, and the Cloud/SNI +proxy address and client-route hostnames no longer resolve themselves either, so with resolution +disabled none of them can connect. The driver fails such an attempt with a message naming the +address and the reason, rather than the bare `UnresolvedAddressException` Netty would otherwise +raise. If you disable the resolver, supply already-resolved addresses. + +The same now applies to the two other address sources that used to perform their own JVM DNS lookups: +the Cloud/SNI proxy address and client-route hostnames are expanded by the connection layer too. A +custom Netty resolver applies to them for the first time, and a proxy hostname with several A-records +has all of them tried within a single connection attempt. + +There is **no public API change**: `EndPoint.resolve()` keeps its signature and is not deprecated. +Third-party `EndPoint` implementations keep working unchanged, with one new expectation — an +implementation should return its address as-is rather than looking names up itself, since resolution +now happens in the connection layer and `resolve()` is called from an event loop. + +**If you call `node.getEndPoint().resolve()` yourself, check how you read the host.** Because +resolution moved to the connection layer, that address is no longer always resolved: + +| Deployment | `resolve()` returns | +| --- | --- | +| Nodes discovered from `system.peers`, and the node the control connection is on | resolved, as before | +| Cloud / SNI proxy | the configured proxy hostname, **unresolved** | +| Cloud private-endpoint client routes (nodes that have a route) | the route hostname, **unresolved** | + +For the last two, `InetSocketAddress.getAddress()` now returns `null`, so a call such as +`((InetSocketAddress) node.getEndPoint().resolve()).getAddress().getHostAddress()` throws a +`NullPointerException` where it previously worked. Use `getHostString()` instead: it returns +whichever of a hostname or an IP literal the address carries, for both the resolved and the +unresolved case, and never triggers a reverse lookup. + +As part of this change: + +- `advanced.resolve-contact-points` is deprecated and now has **no effect**; setting it to `true` + logs a warning at startup. Contact points are always kept as unresolved hostnames and expanded at + connection time. An already-resolved `InetSocketAddress` passed programmatically is still used as + provided, with no further expansion. + + If you had set it to `true`, note what that setting used to buy you, since the driver no longer + offers it: each address a contact-point hostname resolved to became its own `Node`, so each had + its own entry in `Metadata.getNodes()` and its own node-level metrics, and the load balancing + policy's query plan advanced address by address. A hostname is now a single `Node`, and its + addresses are tried inside one connection attempt. Resolution + also used to happen once, at startup; it now happens per connection attempt, through Netty's + configured resolver, which is what lets the driver pick up changed DNS records — but it means the + resolver is consulted far more often than before, so the JVM DNS cache settings + (`networkaddress.cache.ttl`) matter more than they used to. +- **The node the control connection is on is identified by the address it is connected to, not by + the contact point it was reached through.** This shows up in node-level metrics: with a + hostname contact point, the node the control connection came up on used to report under a prefix + derived from that hostname (`nodes.cluster_example_com:9042.*`, and the `node` tag likewise) — + which was never really its own name, since any node can end up being the one a contact point + reaches. It now reports under its own address, like every other node + (`nodes.10_0_0_2:9042.*`). If you scraped a node series named after your contact-point hostname, + expect it to move once, to that node's address. Nothing to configure. +- `advanced.control-connection.reconnection.fallback-to-original-contact-points` now defaults to + `true` (previously `false`). This is also the driver's DNS re-resolution path: a metadata node + normally holds an address decoded from the peers table, which is already resolved and so never + picks up a DNS change, and falling back to the original contact points on control-connection + reconnect is what re-reads the records once the live-node plan is exhausted. (The exception is an + `AddressTranslator` that returns a hostname — `SubnetAddressTranslator` does, unless you set + `advanced.address-translator.resolve-addresses = true`. Such an endpoint is re-expanded on every + connection attempt like any other name, so its node is not pinned to one address in the first + place. Its addresses are tried in the resolver's own order rather than shuffled — see below — so + its channels stay on one address in practice, but if that name maps to more than one host the + driver has no way to tell.) Set it to `false` to + restore the previous behavior. Note the cost: + the contact points are appended without being compared against the live-node plan (at plan time + they are still hostnames, while the live nodes are already-resolved IPs), so when DNS has not + changed a reconnection round that exhausts the live nodes retries roughly twice as many addresses. +- **A contact point that no address can identify is now marked down before the connection attempt + completes.** Establishing which node answered — reading `host_id` from `system.local` — happens + while the driver still holds the hostname's other addresses, so a candidate address that cannot + identify itself is rejected like any other per-address failure and the next address of the same + hostname is tried. When every address has been tried and none worked, the driver fires the same + `controlConnectionFailed` event and node-down state change that a refused TCP connection + produces, where previously it moved on to the next contact point without either. Nothing to + change on your side; expect the DOWN event and its log line in a case that used to be silent. +- **`AllNodesFailedException` groups a hostname's failures.** A contact point that resolves to + several addresses contributes one entry per contact point rather than per address, with each + address's failure attached to it as a suppressed exception. If you inspect + `AllNodesFailedException.getAllErrors()`, expect fewer entries carrying more detail; the underlying + causes are all still reachable through `Throwable.getSuppressed()`. +- **A name's addresses are tried in random order, and at most + `advanced.connection.max-candidate-addresses` of them per connection attempt** (new option, + default 5). The shuffle spreads load across a multi-record name's addresses and varies the + starting address between attempts. It applies to every contact point, and to an already-identified + node when its addresses are interchangeable — an SNI proxy or a cloud private-endpoint route, + where the proxy routes by server name so all of its addresses lead to the same node. For those the + driver spread connections across the records before this version too, inside + `SniEndPoint.resolve()`. The order is kept only for a name that may denote *different* hosts, + which is what an `AddressTranslator` can return (`SubnetAddressTranslator` does by default): one + node's connection pool converges on one address there, as it did before, and the remaining + addresses serve as fallback. The cap applies either way + and bounds what one attempt can cost, since every address + tried is a full TCP connect plus handshake — and, with wrong credentials, a rejected login + (authentication failures do not stop the walk: with a multi-record name they may be specific to + the address, e.g. a stale record pointing at a foreign cluster). Where the order is shuffled, + addresses beyond the cap are + not lost — every attempt samples a fresh random subset, so reconnection rounds reach all of them + over time; where it is kept, the cap is a hard limit and records past it are not reached, which is + still more than the single address such a name got before. Set the option to `1` to restore + one-address-per-attempt behavior. +- **The control connection can now decline a node after a successful handshake.** Once the driver + knows which node answered, it re-checks that node against what the load balancing policy and + topology events say about it, and closes the channel if it is IGNORED, forced down or removed, + moving on to the next entry in the plan. Previously a contact point reached this way was an + ephemeral object that no event had ever named, so the check found nothing and the connection was + kept. This matters most with the contact-point fallback above now on by default: if your contact + points resolve to nodes the policy excludes, expect the control connection to keep looking rather + than settle on one of them. The reason is recorded in the resulting `AllNodesFailedException`. +- If you use `TaggingMetricIdGenerator` **and** build the Cloud proxy address yourself with + `new InetSocketAddress("proxy-host", port)` rather than through a secure connect bundle: the + `node` tag for those nodes changes once, from `proxy-host/1.2.3.4:9042` to `proxy-host:9042`. The + driver now keeps the configured proxy address unresolved so it can be re-expanded to every proxy + A-record, and the tag no longer depends on which proxy IP a connection happened to land on — that + stability is the point, but expect a one-time series rename in dashboards. The same one-time + rename applies if you passed the proxy as an IP literal (`1.2.3.4/1.2.3.4:9042` becomes + `1.2.3.4:9042`), which is stored unresolved too so that the endpoint's identity cannot change + underneath it when something asks the address for its reverse-DNS name. `asMetricPrefix()`, and + therefore the default `MetricIdGenerator`, is unaffected. +- For advanced deployments that provide a custom `NettyOptions`: the + `afterBootstrapInitialized()` hook now runs once per logical connection to a node (previously + once per attempt, including protocol-version downgrade retries), and it receives the bootstrap + *before* the driver installs its channel handler — a handler set by the hook is replaced, and + the driver logs a one-time warning if it detects one. Use the hook for channel options, + attributes and `Bootstrap.resolver(...)`; use `afterChannelInitialized()` for pipeline + customization. +- Two `protected` methods that no longer have anything to do are removed. Both are in internal + packages, but they were reachable from a subclass, so a custom subclass overriding either of them + will no longer compile: + - `OptionalLocalDcHelper.checkLocalDatacenterCompatibility(String, Set)` — it warned when a + contact point's datacenter differed from the configured local DC, but contact-point nodes never + get a datacenter assigned, so it compared against `null` and warned unconditionally instead of + on a real mismatch. The separate "configured local DC matches no node" warning is retained and + now covers the intended case. + - `ClientRoutesTopologyMonitor.resolveAddress(String)` — a test seam for the JVM DNS lookup that + the monitor no longer performs, now that route hostnames are resolved by the connection layer. + ### 4.19.0.7 #### Cloud private-endpoint support via client routes From f83d55f3d5216c11e46fced82d9a9844b90f3e09 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 6 Aug 2026 18:17:17 +0200 Subject: [PATCH 9/9] test: cover multi-address resolution against a real cluster (DRIVER-201) MockResolverIT drives the end-to-end fix through a JVM-level InetAddress hook: a hostname that maps to one dead and one live address must still produce a working session. Its multi-address test was one change away from being vacuous. The comment claimed the dead record was tried first because of resolver insertion order, but rotate() sorts candidates by toString() and discards that order; the dead address went first only because the sort is lexicographic. The test now captures ChannelFactory at DEBUG and requires the "trying next address" event, which was proven load-bearing: moving the dead IP to one that sorts last makes it fail in 7s instead of passing in 89s. ClientRoutesIT asserts on host strings rather than resolved IPs, since a client route now stays unresolved until the connection layer expands it. Co-Authored-By: Claude Opus 5 (1M context) --- .../core/clientroutes/ClientRoutesIT.java | 16 ++- .../driver/core/resolver/MockResolverIT.java | 122 +++++++++++++++++- 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java index 9a23f368546..e658f843360 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java @@ -186,12 +186,12 @@ private void requireSystemClientRoutesTable(CqlSession admin) { } private InetSocketAddress tryResolve(ClientRoutesTopologyMonitor handler, UUID hostId) { + // resolve() is an in-memory cache lookup that hands the route hostname over unresolved -- the + // connection layer resolves it -- so the only failure left is the monitor being closed. try { return handler.resolve(hostId); } catch (IllegalStateException e) { return null; - } catch (UnknownHostException e) { - throw new RuntimeException("DNS resolution failed for host_id=" + hostId, e); } } @@ -206,7 +206,9 @@ private NodeClassification classifyNodes(CqlSession session) { NodeClassification result = new NodeClassification(); for (Node node : session.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // getHostString() rather than getAddress().getHostAddress(): a client route is handed over + // unresolved (the connection layer resolves it), so getAddress() is null for proxied nodes. + String ip = addr.getHostString(); UUID hostId = node.getHostId(); boolean connected = node.getOpenConnections() > 0; LOG.info( @@ -319,7 +321,8 @@ private Map collectHostIds(CcmBridge ccm, int nodeCount, String t .build()) { for (Node node : adminSession.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // See classifyNodes(): a client route is unresolved, so getAddress() would be null. + String ip = addr.getHostString(); Integer nodeId = ipToNodeId.get(ip); if (nodeId != null && node.getHostId() != null) { hostIds.put(nodeId, node.getHostId()); @@ -540,7 +543,10 @@ public void should_refresh_routes_after_table_update() throws Exception { () -> { InetSocketAddress resolved = handler.resolve(hostId); assertThat(resolved).isNotNull(); - assertThat(resolved.getAddress().getHostAddress()).isEqualTo(nodeAddr); + // The route is returned unresolved on purpose -- ChannelFactory resolves it through + // Netty's AddressResolverGroup -- so assert on the host string, not getAddress(). + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo(nodeAddr); assertThat(resolved.getPort()).isEqualTo(9042); }); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java index 4e9eefebf63..4f8ba6897f8 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java @@ -25,9 +25,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.CqlSessionBuilder; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; @@ -37,6 +39,7 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.channel.ChannelFactory; import com.datastax.oss.driver.internal.core.config.typesafe.DefaultProgrammaticDriverConfigLoaderBuilder; import java.net.InetSocketAddress; import java.util.Collection; @@ -44,9 +47,12 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.slf4j.Logger; @@ -60,6 +66,47 @@ public class MockResolverIT { private static final int CLUSTER_WAIT_SECONDS = 20; // Maximal wait time for cluster nodes to get up + /** + * A loopback address no node is ever started on, outside the {@code 127.0.1.} prefix CCM hands + * its nodes: a connection attempt that dials it is refused immediately. + */ + private static final String DEAD_ADDRESS = "127.0.0.11"; + + private static final ch.qos.logback.classic.Logger CHANNEL_FACTORY_LOGGER = + (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(ChannelFactory.class); + + private ListAppender channelFactoryAppender; + private Level originalChannelFactoryLevel; + + @Before + public void startCapturingChannelFactoryLogs() { + // ChannelFactory reports a candidate it gave up on at DEBUG, which is the only externally + // visible evidence that the multi-address fallback did any work. + originalChannelFactoryLevel = CHANNEL_FACTORY_LOGGER.getLevel(); + CHANNEL_FACTORY_LOGGER.setLevel(Level.DEBUG); + channelFactoryAppender = new ListAppender<>(); + // ChannelFactory logs from every I/O thread of the session under test, and the assertions read + // the list while those threads are still winding down; ListAppender's own ArrayList is not safe + // for that (appends are unsynchronized, and iterating one would throw). + channelFactoryAppender.list = new CopyOnWriteArrayList<>(); + channelFactoryAppender.start(); + CHANNEL_FACTORY_LOGGER.addAppender(channelFactoryAppender); + } + + @After + public void stopCapturingChannelFactoryLogs() { + CHANNEL_FACTORY_LOGGER.detachAppender(channelFactoryAppender); + channelFactoryAppender.stop(); + CHANNEL_FACTORY_LOGGER.setLevel(originalChannelFactoryLevel); + } + + /** The formatted messages {@code ChannelFactory} logged during the current test. */ + private List channelFactoryLogMessages() { + return channelFactoryAppender.list.stream() + .map(ILoggingEvent::getFormattedMessage) + .collect(Collectors.toList()); + } + private static void waitForAllNodesUp(CqlSession session, int expectedNodes) { Awaitility.await() .atMost(CLUSTER_WAIT_SECONDS, TimeUnit.SECONDS) @@ -84,7 +131,6 @@ public void should_connect_with_mocked_hostname() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -107,19 +153,82 @@ public void should_connect_with_mocked_hostname() { .filter(x -> x.toString().contains("test.cluster.fake")) .collect(Collectors.toSet()); assertThat(filteredNodes).hasSize(1); - InetSocketAddress address = - (InetSocketAddress) filteredNodes.iterator().next().getEndPoint().resolve(); - assertTrue(address.isUnresolved()); + Node node = filteredNodes.iterator().next(); + InetSocketAddress address = (InetSocketAddress) node.getEndPoint().resolve(); + // ChannelFactory pins the control connection's endpoint to the address it actually reached, + // and DefaultTopologyMonitor#buildNodeEndPoint stores that copy for the control node, so + // resolution yields that concrete IP rather than the hostname. + assertFalse(address.isUnresolved()); + assertThat(address.getAddress().getHostAddress()).isEqualTo(ccmBridge.getNodeIpAddress(1)); + // The pinned copy still denotes the same node by hostname though -- it is what the filter + // above matched on -- so metric names do not depend on which IP a connection landed on. + assertThat(node.getEndPoint().asMetricPrefix()).isEqualTo("test_cluster_fake:9042"); } } } + @Test + public void should_connect_when_first_dns_entry_is_non_responsive() { + final int numberOfNodes = 2; + DriverConfigLoader loader = + new DefaultProgrammaticDriverConfigLoaderBuilder() + .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) + .withStringList( + TypedDriverOption.CONTACT_POINTS.getRawOption(), + Collections.singletonList("test.cluster.fake:9042")) + .build(); + + CqlSessionBuilder builder = new CqlSessionBuilder().withConfigLoader(loader); + try (CcmBridge ccmBridge = + CcmBridge.builder().withNodes(numberOfNodes).withIpPrefix("127.0.1.").build()) { + MultimapHostResolverProvider.removeResolverEntries("test.cluster.fake"); + // Nothing is ever started on DEAD_ADDRESS, so it is the dead record. + MultimapHostResolverProvider.addResolverEntry("test.cluster.fake", DEAD_ADDRESS); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(1)); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(2)); + ccmBridge.create(); + ccmBridge.start(); + + // ChannelFactory shuffles a name's expanded addresses per connection attempt, so whether the + // dead record is dialed first is a coin toss (about 1 in 3 here). Every session must connect + // either way; the loop hunts for an iteration that demonstrably dialed the dead record first + // and fell through to a live one, which is the behavior this test exists to pin down. + // Twenty misses in a row would have probability (2/3)^20, about 0.03%. + boolean sawFallback = false; + for (int attempt = 0; attempt < 20 && !sawFallback; attempt++) { + try (CqlSession session = builder.build()) { + waitForAllNodesUp(session, numberOfNodes); + ResultSet rs = session.execute("select * from system.local where key='local'"); + assertThat(rs).isNotNull(); + List rows = rs.all(); + assertThat(rows).hasSize(1); + Collection nodes = session.getMetadata().getNodes().values(); + assertThat(nodes).hasSize(numberOfNodes); + } + sawFallback = + channelFactoryLogMessages().stream() + .anyMatch( + message -> + message.contains(DEAD_ADDRESS) && message.contains("trying next address")); + } + + // The connections above only survived a dead-first ordering because the candidate loop moved + // past the dead record. + assertThat(sawFallback) + .as( + "expected at least one connection attempt to dial %s first and fall through", + DEAD_ADDRESS) + .isTrue(); + } + } + @Test public void replace_cluster_test() { final int numberOfNodes = 3; DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -206,7 +315,6 @@ public void run_replace_test_20_times() { public void cannot_reconnect_with_resolved_socket() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(),