From 817073d07d01781f083d4728efbbe97fd2201542 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 02:55:07 +0300 Subject: [PATCH 01/35] IGNITE-27977 Refactor bytes serialization for DataStreamerRequest The stream receiver was marshalled by hand: the streamer produced the blob, and the processor unmarshalled it with a class loader it had just built. The pair is now an @Marshalled field, so the generated marshaller owns both directions. The class is marked @UseBinaryMarshaller: the receiver is a user class, and the hand-written call used ctx.marshaller(), which is the same schema-aware marshaller the annotation selects. The wire format is unchanged - updaterBytes stays @Order(3). The message stays a DeferredUnmarshalMessage. Its class loader does not come from a carried deployment alone: with forced local deployment it is the grid class loader. The processor therefore passes the loader explicitly and keeps the read inside its own try, so a missing deployment is still answered to the sender instead of leaving it waiting for a timeout. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamProcessor.java | 12 ++++------ .../datastreamer/DataStreamerImpl.java | 11 +-------- .../datastreamer/DataStreamerRequest.java | 23 ++++++++++++------- .../DataStreamerImplSelfTest.java | 2 +- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index 464a74d82ee25..b2b7f85953dbe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.managers.communication.GridIoManager; import org.apache.ignite.internal.managers.communication.GridMessageListener; +import org.apache.ignite.internal.managers.communication.MessageMarshalling; import org.apache.ignite.internal.managers.deployment.GridDeployment; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; @@ -48,7 +49,6 @@ import org.apache.ignite.lang.IgniteClosure; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.marshaller.Marshaller; import org.apache.ignite.stream.StreamReceiver; import org.jetbrains.annotations.Nullable; @@ -68,9 +68,6 @@ public class DataStreamProcessor extends GridProcessorAdapter { /** Data Streamer flusher. */ private final DataStreamerFlusher flusher = new DataStreamerFlusher(); - /** Marshaller. */ - private final Marshaller marsh; - /** * @param ctx Kernal context. */ @@ -86,8 +83,6 @@ public DataStreamProcessor(GridKernalContext ctx) { } }); } - - marsh = ctx.marshaller(); } /** {@inheritDoc} */ @@ -240,7 +235,10 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { StreamReceiver updater; try { - updater = U.unmarshal(marsh, req.updaterBytes(), U.resolveClassLoader(clsLdr, ctx.config())); + // The request carries user classes, so it is read here, with the deployment class loader at hand. + MessageMarshalling.unmarshal(req, ctx, null, U.resolveClassLoader(clsLdr, ctx.config())); + + updater = req.updater(); if (updater != null) ctx.resource().injectGeneric(updater); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 1040f06ddf83b..944193a281351 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -155,9 +155,6 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Cache receiver. */ private StreamReceiver rcvr = ISOLATED_UPDATER; - /** */ - private byte[] updaterBytes; - /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -1943,12 +1940,6 @@ private void submit( if (val != null) val.marshal(cacheObjCtx); } - - if (updaterBytes == null) { - assert rcvr != null; - - updaterBytes = U.marshal(ctx, rcvr); - } } catch (IgniteCheckedException e) { U.error(log, "Failed to marshal.", e); @@ -1995,7 +1986,7 @@ private void submit( reqId, topicId, cacheName, - updaterBytes, + rcvr, entries, true, skipStore, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 88af958a50373..933bdbba9417f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -22,20 +22,24 @@ import java.util.UUID; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.internal.DeferredUnmarshalMessage; +import org.apache.ignite.internal.Marshalled; import org.apache.ignite.internal.Order; import org.apache.ignite.internal.StripedMessage; +import org.apache.ignite.internal.UseBinaryMarshaller; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheUtils; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.plugin.extensions.communication.CacheIdAware; +import org.apache.ignite.stream.StreamReceiver; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.GridTopic.TOPIC_DATASTREAM; -/** */ +/** A batch of streamed entries. The receiver is a user class, hence the deferred unmarshalling. */ +@UseBinaryMarshaller public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAware, StripedMessage { /** */ @Order(0) @@ -49,8 +53,11 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** */ - // TODO: Refactor bytes serialization - IGNITE-27977 + /** Cache receiver. */ + @Marshalled("updaterBytes") + StreamReceiver updater; + + /** Serialized cache receiver. */ @Order(3) byte[] updaterBytes; @@ -112,7 +119,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterBytes Cache receiver. + * @param updater Cache receiver. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -130,7 +137,7 @@ public DataStreamerRequest( long reqId, IgniteUuid resTopicId, @Nullable String cacheName, - byte[] updaterBytes, + StreamReceiver updater, Collection entries, boolean ignoreDepOwnership, boolean skipStore, @@ -149,7 +156,7 @@ public DataStreamerRequest( this.reqId = reqId; this.resTopicId = resTopicId; this.cacheName = cacheName; - this.updaterBytes = updaterBytes; + this.updater = updater; this.entries = entries; this.ignoreDepOwnership = ignoreDepOwnership; this.skipStore = skipStore; @@ -180,8 +187,8 @@ String cacheName() { } /** @return Updater. */ - byte[] updaterBytes() { - return updaterBytes; + StreamReceiver updater() { + return updater; } /** @return Entries to update. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 2bc777180e0ff..f8ade7da47394 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -692,7 +692,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { req.requestId(), req.resTopicId, req.cacheName(), - req.updaterBytes(), + req.updater(), req.entries(), req.ignoreDeploymentOwnership(), req.skipStore(), From e3ba9b097bc8ef1f73bf8bfffc8de5fcc4cbc97d Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 03:15:35 +0300 Subject: [PATCH 02/35] IGNITE-27977 Simplify the wording around the deferred read The class javadoc and the comment at the read said the same thing twice. The javadoc now states what the message is, and the comment states why the read waits for this point. Co-Authored-By: Claude Opus 5 --- .../internal/processors/datastreamer/DataStreamProcessor.java | 2 +- .../internal/processors/datastreamer/DataStreamerRequest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index b2b7f85953dbe..6e2d787a893b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -235,7 +235,7 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { StreamReceiver updater; try { - // The request carries user classes, so it is read here, with the deployment class loader at hand. + // Read here, not on the inbound pass: the deployment class loader is known only at this point. MessageMarshalling.unmarshal(req, ctx, null, U.resolveClassLoader(clsLdr, ctx.config())); updater = req.updater(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 933bdbba9417f..a5f056e5be852 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -38,7 +38,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_DATASTREAM; -/** A batch of streamed entries. The receiver is a user class, hence the deferred unmarshalling. */ +/** Batch of streamed entries. The receiver is a user class. */ @UseBinaryMarshaller public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAware, StripedMessage { /** */ From bf633f0da4df8537480dd68f6b3f24797355af87 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 03:26:42 +0300 Subject: [PATCH 03/35] IGNITE-27977 Keep the receiver out of the message toString The field holds a user object, and the request is printed on the sending side under debug logging. The blob it replaced printed as bytes, and GridToStringBuilder rethrows whatever a field toString throws, so a user toString could now break the logging path. GridJobExecuteRequest excludes its user objects the same way. Co-Authored-By: Claude Opus 5 --- .../internal/processors/datastreamer/DataStreamerRequest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index a5f056e5be852..c9b54dcce8bd4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.UseBinaryMarshaller; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheUtils; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteUuid; @@ -53,7 +54,8 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache receiver. */ + /** Cache receiver. A user object, kept out of the message {@code toString()}. */ + @GridToStringExclude @Marshalled("updaterBytes") StreamReceiver updater; From d92c10517a1b090995dc9c298b04ec5895f8d66d Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 03:51:50 +0300 Subject: [PATCH 04/35] IGNITE-27977 Marshal the receiver once per streamer Handing the message the receiver instead of its bytes cost a marshal per batch, where the streamer used to marshal once and reuse the result. Measured on one node: 281 ns for IsolatedUpdater, 810 ns for StreamTransformer.from(ep), 9.3 us for a receiver holding 10K of state, against 76 us to build and marshal a batch of 512 entries - up to 12% of a batch, and a larger share of a small one. The streamer keeps the bytes the generated marshaller produced for the first request and hands them to the next one, which the marshaller then keeps instead of producing its own. It reuses a result rather than deciding how to obtain it, so the marshaller stays the one codegen picks. The bytes are paired with the receiver they belong to, so a receiver replaced mid-stream invalidates them by itself - no separate cache reset that a concurrent send could race with. This also closes the older mismatch, where the cache was never invalidated at all and a replaced receiver took effect locally but not remotely. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamerImpl.java | 21 ++++++++- .../datastreamer/DataStreamerRequest.java | 16 +++++++ .../DataStreamerImplSelfTest.java | 47 +++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 944193a281351..c21bd94a98474 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -104,6 +104,7 @@ import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -155,6 +156,9 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Cache receiver. */ private StreamReceiver rcvr = ISOLATED_UPDATER; + /** A receiver paired with the bytes the marshaller produced for it; a replaced receiver invalidates the pair. */ + private volatile T2, byte[]> marshalledRcvr; + /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -1982,11 +1986,13 @@ private void submit( if (topVer == null) topVer = ctx.cache().context().exchange().readyAffinityVersion(); + StreamReceiver rcvr0 = rcvr; + DataStreamerRequest req = new DataStreamerRequest( reqId, topicId, cacheName, - rcvr, + rcvr0, entries, true, skipStore, @@ -1998,11 +2004,22 @@ private void submit( dep != null ? dep.classLoaderId() : null, dep == null, topVer, - (rcvr == ISOLATED_UPDATER) ? partId : NO_STRIPE); + (rcvr0 == ISOLATED_UPDATER) ? partId : NO_STRIPE); + + // Every batch carries the same receiver, so it is marshalled once: the message is handed the bytes of + // an earlier request, and yields its own when it is the first to be marshalled. + T2, byte[]> marshalled = marshalledRcvr; + + byte[] rcvrBytes = marshalled != null && marshalled.get1() == rcvr0 ? marshalled.get2() : null; + + req.updaterBytes(rcvrBytes); try { ctx.io().sendToGridTopic(node, TOPIC_DATASTREAM, req, plc); + if (rcvrBytes == null) + marshalledRcvr = new T2<>(rcvr0, req.updaterBytes()); + if (log.isDebugEnabled()) log.debug("Sent request to node [nodeId=" + node.id() + ", req=" + req + ']'); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index c9b54dcce8bd4..6538752a23481 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -193,6 +193,22 @@ String cacheName() { return updater; } + /** @return Serialized updater, {@code null} until the message is marshalled. */ + byte[] updaterBytes() { + return updaterBytes; + } + + /** + * Hands the message the serialized form of its updater, so the marshaller keeps it instead of producing its own. + * The receiver of a streamer does not change between batches, hence the sender marshals it once and passes the + * result on. + * + * @param updaterBytes Serialized updater taken from an already marshalled request. + */ + void updaterBytes(byte[] updaterBytes) { + this.updaterBytes = updaterBytes; + } + /** @return Entries to update. */ Collection entries() { return entries; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index f8ade7da47394..d68b9150ec8ff 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; @@ -88,6 +89,9 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Indicates whether we need to make the topology stale */ private static boolean needStaleTop = false; + /** Collects the serialized receiver of every streamer request sent, when set. */ + private static volatile Collection sentUpdaterBytes; + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { super.afterTest(); @@ -142,6 +146,42 @@ public void testCloseWithCancellation() throws Exception { assertTrue(fut.isDone()); } + /** + * The receiver does not change between batches, so it is marshalled once: every request carries the very bytes + * produced for the first one. + * + * @throws Exception If failed. + */ + @Test + public void testReceiverMarshalledOncePerStreamer() throws Exception { + cnt = 0; + + startGrids(2); + + Collection sent = new ConcurrentLinkedQueue<>(); + + sentUpdaterBytes = sent; + + try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + ldr.perNodeBufferSize(1); + + for (int i = 0; i < KEYS_COUNT; i++) + ldr.addData(i, i); + } + finally { + sentUpdaterBytes = null; + } + + assertTrue("Expected more than one request to a remote node, got " + sent.size(), sent.size() > 1); + + byte[] first = F.first(sent); + + assertNotNull(first); + + for (byte[] bytes : sent) + assertTrue("The receiver was marshalled more than once", first == bytes); + } + /** * Test inconsistency log warning of the streamer. Default receiver goes first and is set again after a consistent * receiver. The warning must appear only once. @@ -670,6 +710,13 @@ private CacheConfiguration cacheConfiguration() { private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { + Collection updaterBytes = sentUpdaterBytes; + + // The message is already marshalled at this point, so the serialized receiver is in place. + if (updaterBytes != null && msg instanceof GridIoMessage + && ((GridIoMessage)msg).message() instanceof DataStreamerRequest) + updaterBytes.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updaterBytes()); + // Send stale topology only in the first request to avoid indefinitely getting failures. if (needStaleTop) { if (msg instanceof GridIoMessage) { From 29cf297d09b742eb595fdc35b92bdfe9056a1aca Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 04:09:04 +0300 Subject: [PATCH 05/35] IGNITE-27977 Carry the receiver in a message of its own Holding the receiver in the request meant its serialized form belonged to the request, while the object belonged to the streamer. A request is one batch, the receiver lives for the whole stream, so keeping the bytes cost either a marshal per batch or a cache beside the streamer - a cache that had to be invalidated by hand and published safely. The receiver now travels in StreamReceiverMessage, where the object and its bytes sit together and live exactly as long as the receiver does. The streamer holds one instance and puts it into every request, so the generated marshaller fills the bytes for the first batch and the rest find them already there. Replacing the receiver builds another instance, which invalidates the old bytes by construction. This also removes an older race: the receiver field was mutated from the user thread and read by the sending ones without being volatile. The wire format of the request changes: field 3 is now a nested message rather than a byte array. Co-Authored-By: Claude Opus 5 --- .../ignite/internal/CoreMessagesProvider.java | 2 + .../datastreamer/DataStreamerImpl.java | 47 +++++++--------- .../datastreamer/DataStreamerRequest.java | 31 ++--------- .../datastreamer/StreamReceiverMessage.java | 55 +++++++++++++++++++ .../DataStreamerImplSelfTest.java | 26 ++++----- 5 files changed, 93 insertions(+), 68 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java index a460ed789d24b..66be5c98df724 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java @@ -210,6 +210,7 @@ import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry; import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest; import org.apache.ignite.internal.processors.datastreamer.DataStreamerResponse; +import org.apache.ignite.internal.processors.datastreamer.StreamReceiverMessage; import org.apache.ignite.internal.processors.marshaller.MappedName; import org.apache.ignite.internal.processors.marshaller.MappingAcceptedMessage; import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage; @@ -662,6 +663,7 @@ public CoreMessagesProvider(Marshaller dfltMarsh, Marshaller schemaAwareMarsh) { register(DataStreamerEntry.class); register(DataStreamerRequest.class); register(DataStreamerResponse.class); + register(StreamReceiverMessage.class); // [11900 - 12000]: Metrics, monitoring messages. msgIdx = 11900; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index c21bd94a98474..364d57c2f5f04 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -104,7 +104,6 @@ import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -153,11 +152,8 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; - /** Cache receiver. */ - private StreamReceiver rcvr = ISOLATED_UPDATER; - - /** A receiver paired with the bytes the marshaller produced for it; a replaced receiver invalidates the pair. */ - private volatile T2, byte[]> marshalledRcvr; + /** Cache receiver, in the message that carries it to the remote nodes. */ + private volatile StreamReceiverMessage rcvrMsg = new StreamReceiverMessage(ISOLATED_UPDATER); /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -490,12 +486,18 @@ public IgniteInternalFuture internalFuture() { @Override public void receiver(StreamReceiver rcvr) { A.notNull(rcvr, "rcvr"); - this.rcvr = rcvr; + rcvrMsg = new StreamReceiverMessage(rcvr); + } + + /** @return Cache receiver. */ + @SuppressWarnings("unchecked") + private StreamReceiver rcvr() { + return (StreamReceiver)rcvrMsg.receiver(); } /** {@inheritDoc} */ @Override public boolean allowOverwrite() { - return rcvr != ISOLATED_UPDATER; + return rcvr() != ISOLATED_UPDATER; } /** {@inheritDoc} */ @@ -508,7 +510,7 @@ public IgniteInternalFuture internalFuture() { if (node == null) throw new CacheException("Failed to get node for cache: " + cacheName); - rcvr = allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER; + rcvrMsg = new StreamReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); } /** {@inheritDoc} */ @@ -656,7 +658,7 @@ public IgniteFuture addDataInternal(Collection e lock(false); - if (rcvr instanceof IsolatedUpdater && inconsistencyWarned.compareAndSet(false, true)) + if (rcvr() instanceof IsolatedUpdater && inconsistencyWarned.compareAndSet(false, true)) log.warning(WRN_INCONSISTENT_UPDATES); try { @@ -890,9 +892,9 @@ private void load0( if (cacheObjCtx.addDeploymentInfo()) jobPda = new DataStreamerPda(key.value(cacheObjCtx, false), entry.getValue() != null ? entry.getValue().value(cacheObjCtx, false) : null, - rcvr); - else if (rcvr != null) - jobPda = new DataStreamerPda(rcvr); + rcvr()); + else if (rcvr() != null) + jobPda = new DataStreamerPda(rcvr()); initPda = false; } @@ -1851,7 +1853,7 @@ else if (!topFut.isDone()) false, skipStore, keepBinary, - rcvr), + rcvr()), plc); locFuts.add(callFut); @@ -1986,13 +1988,13 @@ private void submit( if (topVer == null) topVer = ctx.cache().context().exchange().readyAffinityVersion(); - StreamReceiver rcvr0 = rcvr; + StreamReceiverMessage rcvrMsg0 = rcvrMsg; DataStreamerRequest req = new DataStreamerRequest( reqId, topicId, cacheName, - rcvr0, + rcvrMsg0, entries, true, skipStore, @@ -2004,22 +2006,11 @@ private void submit( dep != null ? dep.classLoaderId() : null, dep == null, topVer, - (rcvr0 == ISOLATED_UPDATER) ? partId : NO_STRIPE); - - // Every batch carries the same receiver, so it is marshalled once: the message is handed the bytes of - // an earlier request, and yields its own when it is the first to be marshalled. - T2, byte[]> marshalled = marshalledRcvr; - - byte[] rcvrBytes = marshalled != null && marshalled.get1() == rcvr0 ? marshalled.get2() : null; - - req.updaterBytes(rcvrBytes); + (rcvrMsg0.receiver() == ISOLATED_UPDATER) ? partId : NO_STRIPE); try { ctx.io().sendToGridTopic(node, TOPIC_DATASTREAM, req, plc); - if (rcvrBytes == null) - marshalledRcvr = new T2<>(rcvr0, req.updaterBytes()); - if (log.isDebugEnabled()) log.debug("Sent request to node [nodeId=" + node.id() + ", req=" + req + ']'); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 6538752a23481..abea26da3d1b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -22,10 +22,8 @@ import java.util.UUID; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.internal.DeferredUnmarshalMessage; -import org.apache.ignite.internal.Marshalled; import org.apache.ignite.internal.Order; import org.apache.ignite.internal.StripedMessage; -import org.apache.ignite.internal.UseBinaryMarshaller; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheUtils; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -39,8 +37,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_DATASTREAM; -/** Batch of streamed entries. The receiver is a user class. */ -@UseBinaryMarshaller +/** Batch of streamed entries. The receiver it carries is a user class, hence the deferred unmarshalling. */ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAware, StripedMessage { /** */ @Order(0) @@ -56,12 +53,8 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa /** Cache receiver. A user object, kept out of the message {@code toString()}. */ @GridToStringExclude - @Marshalled("updaterBytes") - StreamReceiver updater; - - /** Serialized cache receiver. */ @Order(3) - byte[] updaterBytes; + StreamReceiverMessage updater; /** Entries to update. */ @Order(4) @@ -139,7 +132,7 @@ public DataStreamerRequest( long reqId, IgniteUuid resTopicId, @Nullable String cacheName, - StreamReceiver updater, + StreamReceiverMessage updater, Collection entries, boolean ignoreDepOwnership, boolean skipStore, @@ -190,23 +183,7 @@ String cacheName() { /** @return Updater. */ StreamReceiver updater() { - return updater; - } - - /** @return Serialized updater, {@code null} until the message is marshalled. */ - byte[] updaterBytes() { - return updaterBytes; - } - - /** - * Hands the message the serialized form of its updater, so the marshaller keeps it instead of producing its own. - * The receiver of a streamer does not change between batches, hence the sender marshals it once and passes the - * result on. - * - * @param updaterBytes Serialized updater taken from an already marshalled request. - */ - void updaterBytes(byte[] updaterBytes) { - this.updaterBytes = updaterBytes; + return updater != null ? updater.receiver() : null; } /** @return Entries to update. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java new file mode 100644 index 0000000000000..08423747a3da2 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.datastreamer; + +import org.apache.ignite.internal.Marshalled; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.UseBinaryMarshaller; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.stream.StreamReceiver; + +/** + * The receiver of a streamer on its way to the nodes that own the data: a user object here, its serialized form on + * the wire. One instance serves every batch of a streamer, so the receiver is marshalled once and the batches share + * the result; a streamer given another receiver builds another instance. + */ +@UseBinaryMarshaller +public class StreamReceiverMessage implements Message { + /** */ + @Marshalled("rcvrBytes") + StreamReceiver rcvr; + + /** Serialized {@link #rcvr}, written by whichever batch is marshalled first and read by the rest. */ + @Order(0) + volatile byte[] rcvrBytes; + + /** Empty constructor. */ + public StreamReceiverMessage() { + // No-op. + } + + /** @param rcvr Receiver. */ + StreamReceiverMessage(StreamReceiver rcvr) { + this.rcvr = rcvr; + } + + /** @return Receiver. */ + StreamReceiver receiver() { + return rcvr; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index d68b9150ec8ff..86453aa9e9ccb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -89,8 +89,8 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Indicates whether we need to make the topology stale */ private static boolean needStaleTop = false; - /** Collects the serialized receiver of every streamer request sent, when set. */ - private static volatile Collection sentUpdaterBytes; + /** Collects the receiver carrier of every streamer request sent, when set. */ + private static volatile Collection sentReceivers; /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { @@ -158,9 +158,9 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { startGrids(2); - Collection sent = new ConcurrentLinkedQueue<>(); + Collection sent = new ConcurrentLinkedQueue<>(); - sentUpdaterBytes = sent; + sentReceivers = sent; try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { ldr.perNodeBufferSize(1); @@ -169,17 +169,17 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { ldr.addData(i, i); } finally { - sentUpdaterBytes = null; + sentReceivers = null; } assertTrue("Expected more than one request to a remote node, got " + sent.size(), sent.size() > 1); - byte[] first = F.first(sent); + StreamReceiverMessage first = F.first(sent); - assertNotNull(first); + assertNotNull(first.rcvrBytes); - for (byte[] bytes : sent) - assertTrue("The receiver was marshalled more than once", first == bytes); + for (StreamReceiverMessage rcvr : sent) + assertTrue("The receiver was marshalled more than once", first.rcvrBytes == rcvr.rcvrBytes); } /** @@ -710,12 +710,12 @@ private CacheConfiguration cacheConfiguration() { private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { - Collection updaterBytes = sentUpdaterBytes; + Collection rcvrs = sentReceivers; // The message is already marshalled at this point, so the serialized receiver is in place. - if (updaterBytes != null && msg instanceof GridIoMessage + if (rcvrs != null && msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof DataStreamerRequest) - updaterBytes.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updaterBytes()); + rcvrs.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updater); // Send stale topology only in the first request to avoid indefinitely getting failures. if (needStaleTop) { @@ -739,7 +739,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { req.requestId(), req.resTopicId, req.cacheName(), - req.updater(), + new StreamReceiverMessage(req.updater()), req.entries(), req.ignoreDeploymentOwnership(), req.skipStore(), From d3d15aeb023ad6067d8f75ec408d4082baab78f3 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 04:19:41 +0300 Subject: [PATCH 06/35] IGNITE-27977 Name the receiver accessors after the setter The streamer exposes receiver(StreamReceiver) yet read the field back through rcvr(), and the request named its carrier field after what the getter returns rather than after what it holds. Paired accessors in these classes share a name - allowOverwrite(), skipStore(), keepBinary() - so the getter is receiver() now, and the carrier is updaterMsg. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerImpl.java | 14 +++++++------- .../datastreamer/DataStreamerRequest.java | 12 ++++++------ .../datastreamer/DataStreamerImplSelfTest.java | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 364d57c2f5f04..106357af751ca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -491,13 +491,13 @@ public IgniteInternalFuture internalFuture() { /** @return Cache receiver. */ @SuppressWarnings("unchecked") - private StreamReceiver rcvr() { + private StreamReceiver receiver() { return (StreamReceiver)rcvrMsg.receiver(); } /** {@inheritDoc} */ @Override public boolean allowOverwrite() { - return rcvr() != ISOLATED_UPDATER; + return receiver() != ISOLATED_UPDATER; } /** {@inheritDoc} */ @@ -658,7 +658,7 @@ public IgniteFuture addDataInternal(Collection e lock(false); - if (rcvr() instanceof IsolatedUpdater && inconsistencyWarned.compareAndSet(false, true)) + if (receiver() instanceof IsolatedUpdater && inconsistencyWarned.compareAndSet(false, true)) log.warning(WRN_INCONSISTENT_UPDATES); try { @@ -892,9 +892,9 @@ private void load0( if (cacheObjCtx.addDeploymentInfo()) jobPda = new DataStreamerPda(key.value(cacheObjCtx, false), entry.getValue() != null ? entry.getValue().value(cacheObjCtx, false) : null, - rcvr()); - else if (rcvr() != null) - jobPda = new DataStreamerPda(rcvr()); + receiver()); + else if (receiver() != null) + jobPda = new DataStreamerPda(receiver()); initPda = false; } @@ -1853,7 +1853,7 @@ else if (!topFut.isDone()) false, skipStore, keepBinary, - rcvr()), + receiver()), plc); locFuts.add(callFut); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index abea26da3d1b0..9de9ea1af43ea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -51,10 +51,10 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache receiver. A user object, kept out of the message {@code toString()}. */ + /** Cache receiver, in the message that carries it. Holds a user object, hence out of {@code toString()}. */ @GridToStringExclude @Order(3) - StreamReceiverMessage updater; + StreamReceiverMessage updaterMsg; /** Entries to update. */ @Order(4) @@ -114,7 +114,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updater Cache receiver. + * @param updaterMsg Cache receiver, in the message that carries it. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -132,7 +132,7 @@ public DataStreamerRequest( long reqId, IgniteUuid resTopicId, @Nullable String cacheName, - StreamReceiverMessage updater, + StreamReceiverMessage updaterMsg, Collection entries, boolean ignoreDepOwnership, boolean skipStore, @@ -151,7 +151,7 @@ public DataStreamerRequest( this.reqId = reqId; this.resTopicId = resTopicId; this.cacheName = cacheName; - this.updater = updater; + this.updaterMsg = updaterMsg; this.entries = entries; this.ignoreDepOwnership = ignoreDepOwnership; this.skipStore = skipStore; @@ -183,7 +183,7 @@ String cacheName() { /** @return Updater. */ StreamReceiver updater() { - return updater != null ? updater.receiver() : null; + return updaterMsg != null ? updaterMsg.receiver() : null; } /** @return Entries to update. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 86453aa9e9ccb..7cde572f5bf87 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -715,7 +715,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { // The message is already marshalled at this point, so the serialized receiver is in place. if (rcvrs != null && msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof DataStreamerRequest) - rcvrs.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updater); + rcvrs.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updaterMsg); // Send stale topology only in the first request to avoid indefinitely getting failures. if (needStaleTop) { From 7b82b613132bfcf37a365e8da42ad622d6b31470 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 04:24:34 +0300 Subject: [PATCH 07/35] IGNITE-27977 Say why the receiver bytes are volatile The field is written by the batch that is marshalled first and read by the rest, and those batches leave on different threads. Without the keyword a reader could see the reference before the contents, skip the marshalling and send a half-written array. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/StreamReceiverMessage.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java index 08423747a3da2..076bd8fe384e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java @@ -34,7 +34,11 @@ public class StreamReceiverMessage implements Message { @Marshalled("rcvrBytes") StreamReceiver rcvr; - /** Serialized {@link #rcvr}, written by whichever batch is marshalled first and read by the rest. */ + /** + * Serialized {@link #rcvr}, written by whichever batch is marshalled first and read by the rest. Those batches + * leave on different threads, hence the {@code volatile}: a reader seeing the reference before the contents would + * skip the marshalling and send a half-written array. + */ @Order(0) volatile byte[] rcvrBytes; From 491bdcecf9489a85364aad19a735f9c96e9d88b6 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 04:43:29 +0300 Subject: [PATCH 08/35] IGNITE-27977 Drop the local copy the test did not need The collection was held in a local only because the test cleared the static field in a finally block, before the assertions ran. Clearing it in afterTest, next to the other static cleanup of this class, removes both the local and the try/finally. The local inside the SPI stays and is now explained: it reads the volatile field once, since the field is cleared while nodes that are still stopping keep sending through it. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamerImplSelfTest.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 7cde572f5bf87..6b8b73ad82c07 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -98,6 +98,8 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { stopAllGrids(); + sentReceivers = null; + // Unbinds the log listeners from single static log instance. U.>field(DataStreamerImpl.class, "logRef").set(null); GridTestUtils.setFieldValue(null, DataStreamerImpl.class, "log", null); @@ -158,9 +160,7 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { startGrids(2); - Collection sent = new ConcurrentLinkedQueue<>(); - - sentReceivers = sent; + sentReceivers = new ConcurrentLinkedQueue<>(); try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { ldr.perNodeBufferSize(1); @@ -168,17 +168,15 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { for (int i = 0; i < KEYS_COUNT; i++) ldr.addData(i, i); } - finally { - sentReceivers = null; - } - assertTrue("Expected more than one request to a remote node, got " + sent.size(), sent.size() > 1); + assertTrue("Expected more than one request to a remote node, got " + sentReceivers.size(), + sentReceivers.size() > 1); - StreamReceiverMessage first = F.first(sent); + StreamReceiverMessage first = F.first(sentReceivers); assertNotNull(first.rcvrBytes); - for (StreamReceiverMessage rcvr : sent) + for (StreamReceiverMessage rcvr : sentReceivers) assertTrue("The receiver was marshalled more than once", first.rcvrBytes == rcvr.rcvrBytes); } @@ -710,6 +708,7 @@ private CacheConfiguration cacheConfiguration() { private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { + // Read once: the field is cleared after the test, while nodes still stopping send through this SPI. Collection rcvrs = sentReceivers; // The message is already marshalled at this point, so the serialized receiver is in place. From 0b9c412f5ba349c12d95d1cf25134bf68e679d21 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 04:46:55 +0300 Subject: [PATCH 09/35] IGNITE-27977 Clear the collected receivers before the test, not after Clearing after the test made the field nullable, which cost a null check and a local copy in the SPI, and my comment there claimed a race that did not exist - afterTest clears the field once the grids are already stopped. A final collection cleared in beforeTest gives each test the same clean start with none of that. Co-Authored-By: Claude Opus 5 --- .../DataStreamerImplSelfTest.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 6b8b73ad82c07..8f2a737b835c7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -89,8 +89,15 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Indicates whether we need to make the topology stale */ private static boolean needStaleTop = false; - /** Collects the receiver carrier of every streamer request sent, when set. */ - private static volatile Collection sentReceivers; + /** Receiver carriers of the streamer requests sent since the current test started. */ + private static final Collection sentReceivers = new ConcurrentLinkedQueue<>(); + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + sentReceivers.clear(); + } /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { @@ -98,8 +105,6 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { stopAllGrids(); - sentReceivers = null; - // Unbinds the log listeners from single static log instance. U.>field(DataStreamerImpl.class, "logRef").set(null); GridTestUtils.setFieldValue(null, DataStreamerImpl.class, "log", null); @@ -160,8 +165,6 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { startGrids(2); - sentReceivers = new ConcurrentLinkedQueue<>(); - try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { ldr.perNodeBufferSize(1); @@ -708,13 +711,9 @@ private CacheConfiguration cacheConfiguration() { private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { - // Read once: the field is cleared after the test, while nodes still stopping send through this SPI. - Collection rcvrs = sentReceivers; - // The message is already marshalled at this point, so the serialized receiver is in place. - if (rcvrs != null && msg instanceof GridIoMessage - && ((GridIoMessage)msg).message() instanceof DataStreamerRequest) - rcvrs.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updaterMsg); + if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof DataStreamerRequest) + sentReceivers.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updaterMsg); // Send stale topology only in the first request to avoid indefinitely getting failures. if (needStaleTop) { From 36c67f23ff823ea40870d3dd84fb7cd92a62334a Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Sun, 9 Aug 2026 05:04:20 +0300 Subject: [PATCH 10/35] IGNITE-27977 Self-review touch-ups Read the receiver once where the deployment aware is built, instead of calling the getter three times in a row; keep the explicit type argument on individual() that the rewrite had dropped; unwrap the sent message once in the test SPI; and say in the request javadoc that the excluded field carries a user object rather than being one. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerImpl.java | 10 ++++++---- .../processors/datastreamer/DataStreamerRequest.java | 2 +- .../datastreamer/DataStreamerImplSelfTest.java | 6 ++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 106357af751ca..a8bc307096e4b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -510,7 +510,7 @@ private StreamReceiver receiver() { if (node == null) throw new CacheException("Failed to get node for cache: " + cacheName); - rcvrMsg = new StreamReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); + rcvrMsg = new StreamReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); } /** {@inheritDoc} */ @@ -889,12 +889,14 @@ private void load0( assert key != null; if (initPda) { + StreamReceiver rcvr = receiver(); + if (cacheObjCtx.addDeploymentInfo()) jobPda = new DataStreamerPda(key.value(cacheObjCtx, false), entry.getValue() != null ? entry.getValue().value(cacheObjCtx, false) : null, - receiver()); - else if (receiver() != null) - jobPda = new DataStreamerPda(receiver()); + rcvr); + else if (rcvr != null) + jobPda = new DataStreamerPda(rcvr); initPda = false; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 9de9ea1af43ea..f846c465599a3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -51,7 +51,7 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache receiver, in the message that carries it. Holds a user object, hence out of {@code toString()}. */ + /** Cache receiver, in the message that carries it. Out of {@code toString()}: it is a user object. */ @GridToStringExclude @Order(3) StreamReceiverMessage updaterMsg; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 8f2a737b835c7..5e350aafc1f95 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -711,9 +711,11 @@ private CacheConfiguration cacheConfiguration() { private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { + Message sentMsg = msg instanceof GridIoMessage ? ((GridIoMessage)msg).message() : null; + // The message is already marshalled at this point, so the serialized receiver is in place. - if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof DataStreamerRequest) - sentReceivers.add(((DataStreamerRequest)((GridIoMessage)msg).message()).updaterMsg); + if (sentMsg instanceof DataStreamerRequest) + sentReceivers.add(((DataStreamerRequest)sentMsg).updaterMsg); // Send stale topology only in the first request to avoid indefinitely getting failures. if (needStaleTop) { From c4ca4e66a59f8fb6813fc62c5487eabbaa916a48 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 14:48:20 +0300 Subject: [PATCH 11/35] IGNITE-27977 Review: naming and shorter wording - StreamReceiverMessage -> DataStreamerReceiverMessage, so the name says which streamer it belongs to. - Drop the claim that the updater is a user class: it may as well be one of the updaters Ignite ships. - Shorten the carrier javadoc and stop explaining the data streamer in it. - Say "cache updater" for the request field, and mark the getter @Nullable. - Read rcvrMsg directly instead of through a local. Co-Authored-By: Claude Opus 5 --- .../ignite/internal/CoreMessagesProvider.java | 4 ++-- .../datastreamer/DataStreamerImpl.java | 12 +++++------ ....java => DataStreamerReceiverMessage.java} | 20 ++++++------------- .../datastreamer/DataStreamerRequest.java | 12 +++++------ .../DataStreamerImplSelfTest.java | 8 ++++---- 5 files changed, 23 insertions(+), 33 deletions(-) rename modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/{StreamReceiverMessage.java => DataStreamerReceiverMessage.java} (64%) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java index d8c805cb8631f..6513e174962a2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java @@ -208,9 +208,9 @@ import org.apache.ignite.internal.processors.continuous.StopRoutineAckDiscoveryMessage; import org.apache.ignite.internal.processors.continuous.StopRoutineDiscoveryMessage; import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry; +import org.apache.ignite.internal.processors.datastreamer.DataStreamerReceiverMessage; import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest; import org.apache.ignite.internal.processors.datastreamer.DataStreamerResponse; -import org.apache.ignite.internal.processors.datastreamer.StreamReceiverMessage; import org.apache.ignite.internal.processors.marshaller.MappedName; import org.apache.ignite.internal.processors.marshaller.MappingAcceptedMessage; import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage; @@ -663,7 +663,7 @@ public CoreMessagesProvider(Marshaller dfltMarsh, Marshaller schemaAwareMarsh) { register(DataStreamerEntry.class); register(DataStreamerRequest.class); register(DataStreamerResponse.class); - register(StreamReceiverMessage.class); + register(DataStreamerReceiverMessage.class); // [11900 - 12000]: Metrics, monitoring messages. msgIdx = 11900; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index c3e21db02b3ff..d723ff8adcc1c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -153,7 +153,7 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; /** Cache receiver, in the message that carries it to the remote nodes. */ - private volatile StreamReceiverMessage rcvrMsg = new StreamReceiverMessage(ISOLATED_UPDATER); + private volatile DataStreamerReceiverMessage rcvrMsg = new DataStreamerReceiverMessage(ISOLATED_UPDATER); /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -486,7 +486,7 @@ public IgniteInternalFuture internalFuture() { @Override public void receiver(StreamReceiver rcvr) { A.notNull(rcvr, "rcvr"); - rcvrMsg = new StreamReceiverMessage(rcvr); + rcvrMsg = new DataStreamerReceiverMessage(rcvr); } /** @return Cache receiver. */ @@ -510,7 +510,7 @@ private StreamReceiver receiver() { if (node == null) throw new CacheException("Failed to get node for cache: " + cacheName); - rcvrMsg = new StreamReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); + rcvrMsg = new DataStreamerReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); } /** {@inheritDoc} */ @@ -1990,13 +1990,11 @@ private void submit( if (topVer == null) topVer = ctx.cache().context().exchange().readyAffinityVersion(); - StreamReceiverMessage rcvrMsg0 = rcvrMsg; - DataStreamerRequest req = new DataStreamerRequest( reqId, topicId, cacheName, - rcvrMsg0, + rcvrMsg, entries, true, skipStore, @@ -2005,7 +2003,7 @@ private void submit( dep != null ? jobPda0.deployClass().getName() : null, dep == null, topVer, - (rcvrMsg0.receiver() == ISOLATED_UPDATER) ? partId : NO_STRIPE); + (rcvrMsg.receiver() == ISOLATED_UPDATER) ? partId : NO_STRIPE); try { ctx.io().sendToGridTopic(node, TOPIC_DATASTREAM, req, plc); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java similarity index 64% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index 076bd8fe384e5..275614a7315d7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/StreamReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -23,32 +23,24 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.stream.StreamReceiver; -/** - * The receiver of a streamer on its way to the nodes that own the data: a user object here, its serialized form on - * the wire. One instance serves every batch of a streamer, so the receiver is marshalled once and the batches share - * the result; a streamer given another receiver builds another instance. - */ +/** Cache updater together with its serialized form, shared by the batches of one streamer. */ @UseBinaryMarshaller -public class StreamReceiverMessage implements Message { +public class DataStreamerReceiverMessage implements Message { /** */ @Marshalled("rcvrBytes") StreamReceiver rcvr; - /** - * Serialized {@link #rcvr}, written by whichever batch is marshalled first and read by the rest. Those batches - * leave on different threads, hence the {@code volatile}: a reader seeing the reference before the contents would - * skip the marshalling and send a half-written array. - */ + /** Serialized {@link #rcvr}. Volatile: the batches sharing it are marshalled on different threads. */ @Order(0) volatile byte[] rcvrBytes; - /** Empty constructor. */ - public StreamReceiverMessage() { + /** Empty constructor for serialization purposes. */ + public DataStreamerReceiverMessage() { // No-op. } /** @param rcvr Receiver. */ - StreamReceiverMessage(StreamReceiver rcvr) { + DataStreamerReceiverMessage(StreamReceiver rcvr) { this.rcvr = rcvr; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index be5b4b4d77221..98718c0b08f00 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -35,7 +35,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_DATASTREAM; -/** Batch of streamed entries. The receiver it carries is a user class, hence the deferred unmarshalling. */ +/** Batch of streamed entries. The updater is unmarshalled by the consumer, which has the deployment class loader. */ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAware, StripedMessage { /** */ @Order(0) @@ -49,10 +49,10 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache receiver, in the message that carries it. Out of {@code toString()}: it is a user object. */ + /** Cache updater. */ @GridToStringExclude @Order(3) - StreamReceiverMessage updaterMsg; + DataStreamerReceiverMessage updaterMsg; /** Entries to update. */ @Order(4) @@ -99,7 +99,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterMsg Cache receiver, in the message that carries it. + * @param updaterMsg Cache updater. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -114,7 +114,7 @@ public DataStreamerRequest( long reqId, IgniteUuid resTopicId, @Nullable String cacheName, - StreamReceiverMessage updaterMsg, + DataStreamerReceiverMessage updaterMsg, Collection entries, boolean ignoreDepOwnership, boolean skipStore, @@ -158,7 +158,7 @@ String cacheName() { } /** @return Updater. */ - StreamReceiver updater() { + @Nullable StreamReceiver updater() { return updaterMsg != null ? updaterMsg.receiver() : null; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 4d3c25d2a2d4d..e53458c167a4e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -90,7 +90,7 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { private static boolean needStaleTop = false; /** Receiver carriers of the streamer requests sent since the current test started. */ - private static final Collection sentReceivers = new ConcurrentLinkedQueue<>(); + private static final Collection sentReceivers = new ConcurrentLinkedQueue<>(); /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { @@ -175,11 +175,11 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { assertTrue("Expected more than one request to a remote node, got " + sentReceivers.size(), sentReceivers.size() > 1); - StreamReceiverMessage first = F.first(sentReceivers); + DataStreamerReceiverMessage first = F.first(sentReceivers); assertNotNull(first.rcvrBytes); - for (StreamReceiverMessage rcvr : sentReceivers) + for (DataStreamerReceiverMessage rcvr : sentReceivers) assertTrue("The receiver was marshalled more than once", first.rcvrBytes == rcvr.rcvrBytes); } @@ -739,7 +739,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { req.requestId(), req.resTopicId, req.cacheName(), - new StreamReceiverMessage(req.updater()), + new DataStreamerReceiverMessage(req.updater()), req.entries(), req.ignoreDeploymentOwnership(), req.skipStore(), From 6b645f96e3c974921d3d235ea603184ea895302a Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 16:45:00 +0300 Subject: [PATCH 12/35] IGNITE-27977 Review: trim the carrier javadoc Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerReceiverMessage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index 275614a7315d7..78bf2c54f8623 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -23,7 +23,7 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.stream.StreamReceiver; -/** Cache updater together with its serialized form, shared by the batches of one streamer. */ +/** Cache updater. */ @UseBinaryMarshaller public class DataStreamerReceiverMessage implements Message { /** */ From 771f36ae032eb566764da7bbce0f6d5049ede8a6 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 17:05:26 +0300 Subject: [PATCH 13/35] IGNITE-27977 Stop sending the isolated updater It ships with every node, so serializing it into each request only to deserialize an identical instance on the other side is pure waste - the receiving node can use its own. Requests now carry no updater at all in that case, and the processor falls back to the local ISOLATED_UPDATER. The marshal-once test needed a receiver that actually travels, and both streamer tests now wait for the partition map, since with allowOverwrite a batch only goes to the primary and an unfinished exchange keeps every primary local. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamProcessor.java | 5 ++- .../datastreamer/DataStreamerImpl.java | 10 +++-- .../datastreamer/DataStreamerRequest.java | 6 +-- .../DataStreamerImplSelfTest.java | 41 +++++++++++++++++-- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index fe8fe1ab6d52b..f82a9174b5033 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -235,7 +235,10 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { updater = req.updater(); - if (updater != null) + // The isolated updater is not sent: it ships with this node, so the local one is used instead. + if (updater == null) + updater = DataStreamerImpl.ISOLATED_UPDATER; + else ctx.resource().injectGeneric(updater); } catch (IgniteCheckedException e) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index d723ff8adcc1c..059f617773351 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -146,8 +146,8 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed */ private final Map threadBufMap = new ConcurrentHashMap<>(); - /** Isolated receiver. */ - private static final StreamReceiver ISOLATED_UPDATER = new IsolatedUpdater(); + /** Isolated receiver. Ships with the node, so it is never sent with a request. */ + static final StreamReceiver ISOLATED_UPDATER = new IsolatedUpdater(); /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; @@ -1990,11 +1990,13 @@ private void submit( if (topVer == null) topVer = ctx.cache().context().exchange().readyAffinityVersion(); + boolean isolated = rcvrMsg.receiver() == ISOLATED_UPDATER; + DataStreamerRequest req = new DataStreamerRequest( reqId, topicId, cacheName, - rcvrMsg, + isolated ? null : rcvrMsg, entries, true, skipStore, @@ -2003,7 +2005,7 @@ private void submit( dep != null ? jobPda0.deployClass().getName() : null, dep == null, topVer, - (rcvrMsg.receiver() == ISOLATED_UPDATER) ? partId : NO_STRIPE); + isolated ? partId : NO_STRIPE); try { ctx.io().sendToGridTopic(node, TOPIC_DATASTREAM, req, plc); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 98718c0b08f00..a08882f4fc158 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -49,7 +49,7 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache updater. */ + /** Cache updater, {@code null} for the one the node ships with. */ @GridToStringExclude @Order(3) DataStreamerReceiverMessage updaterMsg; @@ -99,7 +99,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterMsg Cache updater. + * @param updaterMsg Cache updater, {@code null} for the one the node ships with. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -114,7 +114,7 @@ public DataStreamerRequest( long reqId, IgniteUuid resTopicId, @Nullable String cacheName, - DataStreamerReceiverMessage updaterMsg, + @Nullable DataStreamerReceiverMessage updaterMsg, Collection entries, boolean ignoreDepOwnership, boolean skipStore, diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index e53458c167a4e..b88bb1ab60010 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -20,11 +20,11 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; @@ -89,8 +89,8 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Indicates whether we need to make the topology stale */ private static boolean needStaleTop = false; - /** Receiver carriers of the streamer requests sent since the current test started. */ - private static final Collection sentReceivers = new ConcurrentLinkedQueue<>(); + /** Receiver carriers of the streamer requests sent since the current test started; {@code null} for the isolated one. */ + private static final List sentReceivers = Collections.synchronizedList(new ArrayList<>()); /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { @@ -165,7 +165,11 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { startGrids(2); + awaitPartitionMapExchange(); + try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + ldr.receiver(DataStreamerCacheUpdaters.batched()); + ldr.perNodeBufferSize(1); for (int i = 0; i < KEYS_COUNT; i++) @@ -183,6 +187,37 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { assertTrue("The receiver was marshalled more than once", first.rcvrBytes == rcvr.rcvrBytes); } + /** + * The isolated updater ships with every node, so requests carry no updater at all and the data still lands. + * + * @throws Exception If failed. + */ + @Test + public void testIsolatedUpdaterIsNotSent() throws Exception { + cnt = 0; + + startGrids(2); + + awaitPartitionMapExchange(); + + try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + ldr.perNodeBufferSize(1); + + for (int i = 0; i < KEYS_COUNT; i++) + ldr.addData(i, i); + } + + assertTrue("Expected requests to a remote node, got " + sentReceivers.size(), !sentReceivers.isEmpty()); + + for (DataStreamerReceiverMessage rcvr : sentReceivers) + assertNull("The isolated updater was sent with a request", rcvr); + + IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < KEYS_COUNT; i++) + assertEquals(i, cache.get(i)); + } + /** * Test inconsistency log warning of the streamer. Default receiver goes first and is set again after a consistent * receiver. The warning must appear only once. From 7ca61ef08fa8a7d8149111d7c24c2aa633fe0858 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 17:30:05 +0300 Subject: [PATCH 14/35] IGNITE-27977 Drop the carrier for the isolated updater Since the isolated updater is no longer sent, wrapping it into a carrier served nothing: null now means the same thing in the field as it does on the wire - the updater the node ships with. Which also spares the default streamer an object it never uses. The field is read once per request, so the request and its stripe agree on the receiver even if it is replaced mid-stream. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamerImpl.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 059f617773351..365bf0f042647 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -152,8 +152,8 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; - /** Cache receiver, in the message that carries it to the remote nodes. */ - private volatile DataStreamerReceiverMessage rcvrMsg = new DataStreamerReceiverMessage(ISOLATED_UPDATER); + /** Cache receiver, in the message that carries it; {@code null} for {@link #ISOLATED_UPDATER}, which is not sent. */ + private volatile DataStreamerReceiverMessage rcvrMsg; /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -492,12 +492,14 @@ public IgniteInternalFuture internalFuture() { /** @return Cache receiver. */ @SuppressWarnings("unchecked") private StreamReceiver receiver() { - return (StreamReceiver)rcvrMsg.receiver(); + DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; + + return rcvrMsg0 == null ? ISOLATED_UPDATER : (StreamReceiver)rcvrMsg0.receiver(); } /** {@inheritDoc} */ @Override public boolean allowOverwrite() { - return receiver() != ISOLATED_UPDATER; + return rcvrMsg != null; } /** {@inheritDoc} */ @@ -510,7 +512,7 @@ private StreamReceiver receiver() { if (node == null) throw new CacheException("Failed to get node for cache: " + cacheName); - rcvrMsg = new DataStreamerReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); + rcvrMsg = allow ? new DataStreamerReceiverMessage(DataStreamerCacheUpdaters.individual()) : null; } /** {@inheritDoc} */ @@ -658,7 +660,7 @@ public IgniteFuture addDataInternal(Collection e lock(false); - if (receiver() instanceof IsolatedUpdater && inconsistencyWarned.compareAndSet(false, true)) + if (rcvrMsg == null && inconsistencyWarned.compareAndSet(false, true)) log.warning(WRN_INCONSISTENT_UPDATES); try { @@ -1990,13 +1992,14 @@ private void submit( if (topVer == null) topVer = ctx.cache().context().exchange().readyAffinityVersion(); - boolean isolated = rcvrMsg.receiver() == ISOLATED_UPDATER; + // Read once, so the request and the stripe below agree on the receiver. + DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; DataStreamerRequest req = new DataStreamerRequest( reqId, topicId, cacheName, - isolated ? null : rcvrMsg, + rcvrMsg0, entries, true, skipStore, @@ -2005,7 +2008,7 @@ private void submit( dep != null ? jobPda0.deployClass().getName() : null, dep == null, topVer, - isolated ? partId : NO_STRIPE); + rcvrMsg0 == null ? partId : NO_STRIPE); try { ctx.io().sendToGridTopic(node, TOPIC_DATASTREAM, req, plc); From 77f6e4c888c9a200cc8e8935dff0cbb0877e092f Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 17:42:34 +0300 Subject: [PATCH 15/35] IGNITE-27977 Drop comments that repeat the API Why the read happens in the processor is what DeferredUnmarshalMessage documents; what a null updater means now sits on the getter and the field instead of at the call site; and locals named xxx0 are the idiom of this class already (jobPda0 sits two lines above). The rest is shortened. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamProcessor.java | 2 -- .../processors/datastreamer/DataStreamerImpl.java | 5 ++--- .../datastreamer/DataStreamerReceiverMessage.java | 2 +- .../processors/datastreamer/DataStreamerRequest.java | 8 ++++---- .../processors/datastreamer/DataStreamerImplSelfTest.java | 4 ++-- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index f82a9174b5033..03202c355608a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -230,12 +230,10 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { StreamReceiver updater; try { - // Read here, not on the inbound pass: the deployment class loader is known only at this point. MessageMarshalling.unmarshal(req, ctx, null, U.resolveClassLoader(clsLdr, ctx.config())); updater = req.updater(); - // The isolated updater is not sent: it ships with this node, so the local one is used instead. if (updater == null) updater = DataStreamerImpl.ISOLATED_UPDATER; else diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 365bf0f042647..9fff34d81dfa3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -146,13 +146,13 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed */ private final Map threadBufMap = new ConcurrentHashMap<>(); - /** Isolated receiver. Ships with the node, so it is never sent with a request. */ + /** Isolated receiver. */ static final StreamReceiver ISOLATED_UPDATER = new IsolatedUpdater(); /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; - /** Cache receiver, in the message that carries it; {@code null} for {@link #ISOLATED_UPDATER}, which is not sent. */ + /** Cache receiver in its message; {@code null} for {@link #ISOLATED_UPDATER}. */ private volatile DataStreamerReceiverMessage rcvrMsg; /** IO policy resovler for data load request. */ @@ -1992,7 +1992,6 @@ private void submit( if (topVer == null) topVer = ctx.cache().context().exchange().readyAffinityVersion(); - // Read once, so the request and the stripe below agree on the receiver. DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; DataStreamerRequest req = new DataStreamerRequest( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index 78bf2c54f8623..cb2e46f15e00b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -30,7 +30,7 @@ public class DataStreamerReceiverMessage implements Message { @Marshalled("rcvrBytes") StreamReceiver rcvr; - /** Serialized {@link #rcvr}. Volatile: the batches sharing it are marshalled on different threads. */ + /** Serialized {@link #rcvr}. */ @Order(0) volatile byte[] rcvrBytes; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index a08882f4fc158..aeaf4cccab9e9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -35,7 +35,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_DATASTREAM; -/** Batch of streamed entries. The updater is unmarshalled by the consumer, which has the deployment class loader. */ +/** Batch of streamed entries. */ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAware, StripedMessage { /** */ @Order(0) @@ -49,7 +49,7 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache updater, {@code null} for the one the node ships with. */ + /** Cache updater; {@code null} for the isolated one, which every node has. */ @GridToStringExclude @Order(3) DataStreamerReceiverMessage updaterMsg; @@ -99,7 +99,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterMsg Cache updater, {@code null} for the one the node ships with. + * @param updaterMsg Cache updater, {@code null} for the isolated updater. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -157,7 +157,7 @@ String cacheName() { return cacheName; } - /** @return Updater. */ + /** @return Updater, {@code null} for the isolated one. */ @Nullable StreamReceiver updater() { return updaterMsg != null ? updaterMsg.receiver() : null; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index b88bb1ab60010..8d0bea1c2156d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -748,7 +748,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { Message sentMsg = msg instanceof GridIoMessage ? ((GridIoMessage)msg).message() : null; - // The message is already marshalled at this point, so the serialized receiver is in place. + // Already marshalled at this point. if (sentMsg instanceof DataStreamerRequest) sentReceivers.add(((DataStreamerRequest)sentMsg).updaterMsg); @@ -774,7 +774,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { req.requestId(), req.resTopicId, req.cacheName(), - new DataStreamerReceiverMessage(req.updater()), + req.updaterMsg, req.entries(), req.ignoreDeploymentOwnership(), req.skipStore(), From 29fa7753aa353364a32612936da0a307fd3cc215 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 17:48:52 +0300 Subject: [PATCH 16/35] IGNITE-27977 Review: say "default" for the absent updater Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerRequest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index aeaf4cccab9e9..0c8341923d312 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -49,7 +49,7 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache updater; {@code null} for the isolated one, which every node has. */ + /** Cache updater; {@code null} for the default one. */ @GridToStringExclude @Order(3) DataStreamerReceiverMessage updaterMsg; @@ -99,7 +99,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterMsg Cache updater, {@code null} for the isolated updater. + * @param updaterMsg Cache updater, {@code null} for the default one. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -157,7 +157,7 @@ String cacheName() { return cacheName; } - /** @return Updater, {@code null} for the isolated one. */ + /** @return Updater, {@code null} for the default one. */ @Nullable StreamReceiver updater() { return updaterMsg != null ? updaterMsg.receiver() : null; } From 102d49da437faef9249302a8683de3beb61088e4 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 17:51:52 +0300 Subject: [PATCH 17/35] IGNITE-27977 Name the null check instead of commenting it isolated() reads as what it tests, so the places asking "is the default receiver in use" no longer need a note explaining that null means ISOLATED_UPDATER. allowOverwrite() is now its negation, which is what it always was. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 9fff34d81dfa3..9b9fc921364c8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -146,7 +146,7 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed */ private final Map threadBufMap = new ConcurrentHashMap<>(); - /** Isolated receiver. */ + /** Default, isolated receiver. */ static final StreamReceiver ISOLATED_UPDATER = new IsolatedUpdater(); /** Amount of permissions should be available to continue new data processing. */ @@ -497,9 +497,14 @@ private StreamReceiver receiver() { return rcvrMsg0 == null ? ISOLATED_UPDATER : (StreamReceiver)rcvrMsg0.receiver(); } + /** @return {@code True} if the default, isolated receiver is in use. */ + private boolean isolated() { + return rcvrMsg == null; + } + /** {@inheritDoc} */ @Override public boolean allowOverwrite() { - return rcvrMsg != null; + return !isolated(); } /** {@inheritDoc} */ @@ -660,7 +665,7 @@ public IgniteFuture addDataInternal(Collection e lock(false); - if (rcvrMsg == null && inconsistencyWarned.compareAndSet(false, true)) + if (isolated() && inconsistencyWarned.compareAndSet(false, true)) log.warning(WRN_INCONSISTENT_UPDATES); try { From b60b15d74eb8d7f0a67a87fa35710747bec362c4 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 18:11:28 +0300 Subject: [PATCH 18/35] IGNITE-27977 Review: wording of the carrier and the isolated receiver Co-Authored-By: Claude Opus 5 --- .../internal/processors/datastreamer/DataStreamerImpl.java | 4 ++-- .../processors/datastreamer/DataStreamerReceiverMessage.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 9b9fc921364c8..27ac5ce6623d7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -146,7 +146,7 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed */ private final Map threadBufMap = new ConcurrentHashMap<>(); - /** Default, isolated receiver. */ + /** Default, Isolated receiver. */ static final StreamReceiver ISOLATED_UPDATER = new IsolatedUpdater(); /** Amount of permissions should be available to continue new data processing. */ @@ -497,7 +497,7 @@ private StreamReceiver receiver() { return rcvrMsg0 == null ? ISOLATED_UPDATER : (StreamReceiver)rcvrMsg0.receiver(); } - /** @return {@code True} if the default, isolated receiver is in use. */ + /** @return {@code True} if the default, Isolated receiver is in use. */ private boolean isolated() { return rcvrMsg == null; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index cb2e46f15e00b..e1c36cf17cc43 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -23,10 +23,10 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.stream.StreamReceiver; -/** Cache updater. */ +/** DataStreamer cache receiver/updater message. */ @UseBinaryMarshaller public class DataStreamerReceiverMessage implements Message { - /** */ + /** DataStreamer cache receiver/updater. */ @Marshalled("rcvrBytes") StreamReceiver rcvr; From 802ce0c9a8755cefd898a24805cad2c82e9692f6 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 18:18:38 +0300 Subject: [PATCH 19/35] IGNITE-27977 Review: one helper for the two streaming tests Both tests started two nodes, waited for the partition map and streamed the same entries; only the receiver and the assertions differed. The collected carriers are cleared there as well, so the rest of the class no longer pays for a cleanup it does not need. Co-Authored-By: Claude Opus 5 --- .../DataStreamerImplSelfTest.java | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 8d0bea1c2156d..46d63088a5143 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -61,6 +61,7 @@ import org.apache.ignite.testframework.LogListener; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.logging.log4j.core.appender.WriterAppender; +import org.jetbrains.annotations.Nullable; import org.junit.Ignore; import org.junit.Test; @@ -92,13 +93,6 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Receiver carriers of the streamer requests sent since the current test started; {@code null} for the isolated one. */ private static final List sentReceivers = Collections.synchronizedList(new ArrayList<>()); - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - sentReceivers.clear(); - } - /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { super.afterTest(); @@ -154,27 +148,41 @@ public void testCloseWithCancellation() throws Exception { } /** - * The receiver does not change between batches, so it is marshalled once: every request carries the very bytes - * produced for the first one. + * Streams {@link #KEYS_COUNT} entries from the first node, one entry per request, and collects the receiver + * carrier of every request that leaves the node. * + * @param rcvr Receiver to set, or {@code null} to keep the default one. * @throws Exception If failed. */ - @Test - public void testReceiverMarshalledOncePerStreamer() throws Exception { + private void streamToRemoteNode(@Nullable StreamReceiver rcvr) throws Exception { cnt = 0; startGrids(2); awaitPartitionMapExchange(); + sentReceivers.clear(); + try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { - ldr.receiver(DataStreamerCacheUpdaters.batched()); + if (rcvr != null) + ldr.receiver(rcvr); ldr.perNodeBufferSize(1); for (int i = 0; i < KEYS_COUNT; i++) ldr.addData(i, i); } + } + + /** + * The receiver does not change between batches, so it is marshalled once: every request carries the very bytes + * produced for the first one. + * + * @throws Exception If failed. + */ + @Test + public void testReceiverMarshalledOncePerStreamer() throws Exception { + streamToRemoteNode(DataStreamerCacheUpdaters.batched()); assertTrue("Expected more than one request to a remote node, got " + sentReceivers.size(), sentReceivers.size() > 1); @@ -194,18 +202,7 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { */ @Test public void testIsolatedUpdaterIsNotSent() throws Exception { - cnt = 0; - - startGrids(2); - - awaitPartitionMapExchange(); - - try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { - ldr.perNodeBufferSize(1); - - for (int i = 0; i < KEYS_COUNT; i++) - ldr.addData(i, i); - } + streamToRemoteNode(null); assertTrue("Expected requests to a remote node, got " + sentReceivers.size(), !sentReceivers.isEmpty()); From 4f5c613479c7256cb29ef615b1eedc0560405d97 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 18:27:35 +0300 Subject: [PATCH 20/35] IGNITE-27977 Name every built-in updater instead of sending it The isolated one was only half the story: individual, batched and batchedSorted ship with every node too, yet each request serialized one of them and the receiving node deserialized an identical copy. A request now names the built-in updater it wants and carries a serialized one only when it belongs to the user. DataStreamerBuiltInUpdater is that name, and it maps back to the local instance on the other side. The marshal-once test needed a receiver of its own, since the built-in one it used no longer travels, and a new test covers a built-in receiver set explicitly rather than by default. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamProcessor.java | 5 +- .../DataStreamerBuiltInUpdater.java | 65 +++++++++++++++++++ .../datastreamer/DataStreamerImpl.java | 20 +++--- .../datastreamer/DataStreamerRequest.java | 17 +++-- .../DataStreamerImplSelfTest.java | 36 ++++++++-- 5 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index 03202c355608a..d83c630f38595 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -234,9 +234,8 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { updater = req.updater(); - if (updater == null) - updater = DataStreamerImpl.ISOLATED_UPDATER; - else + // Only a user updater arrives with the request and needs its resources injected. + if (req.updaterMsg != null) ctx.resource().injectGeneric(updater); } catch (IgniteCheckedException e) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java new file mode 100644 index 0000000000000..406ac85a9b2d6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.datastreamer; + +import org.apache.ignite.stream.StreamReceiver; +import org.jetbrains.annotations.Nullable; + +/** + * The updaters DataStreamer ships with. Every node has them, so a request names the one it needs instead of carrying + * a serialized copy. + */ +public enum DataStreamerBuiltInUpdater { + /** {@link DataStreamerImpl#ISOLATED_UPDATER}. */ + ISOLATED(DataStreamerImpl.ISOLATED_UPDATER), + + /** {@link DataStreamerCacheUpdaters#individual()}. */ + INDIVIDUAL(DataStreamerCacheUpdaters.individual()), + + /** {@link DataStreamerCacheUpdaters#batched()}. */ + BATCHED(DataStreamerCacheUpdaters.batched()), + + /** {@link DataStreamerCacheUpdaters#batchedSorted()}. */ + BATCHED_SORTED(DataStreamerCacheUpdaters.batchedSorted()); + + /** */ + private final StreamReceiver updater; + + /** @param updater Updater this constant stands for. */ + DataStreamerBuiltInUpdater(StreamReceiver updater) { + this.updater = updater; + } + + /** @return Updater of this node. */ + StreamReceiver updater() { + return updater; + } + + /** + * @param updater Updater to look up. + * @return Constant standing for {@code updater}, or {@code null} when it is not one the streamer ships with. + */ + static @Nullable DataStreamerBuiltInUpdater of(StreamReceiver updater) { + for (DataStreamerBuiltInUpdater builtIn : values()) { + if (builtIn.updater == updater) + return builtIn; + } + + return null; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 27ac5ce6623d7..8200b70eaeae1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -152,8 +152,8 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; - /** Cache receiver in its message; {@code null} for {@link #ISOLATED_UPDATER}. */ - private volatile DataStreamerReceiverMessage rcvrMsg; + /** Cache receiver in its message. */ + private volatile DataStreamerReceiverMessage rcvrMsg = new DataStreamerReceiverMessage(ISOLATED_UPDATER); /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -492,14 +492,12 @@ public IgniteInternalFuture internalFuture() { /** @return Cache receiver. */ @SuppressWarnings("unchecked") private StreamReceiver receiver() { - DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; - - return rcvrMsg0 == null ? ISOLATED_UPDATER : (StreamReceiver)rcvrMsg0.receiver(); + return (StreamReceiver)rcvrMsg.receiver(); } /** @return {@code True} if the default, Isolated receiver is in use. */ private boolean isolated() { - return rcvrMsg == null; + return rcvrMsg.receiver() == ISOLATED_UPDATER; } /** {@inheritDoc} */ @@ -517,7 +515,7 @@ private boolean isolated() { if (node == null) throw new CacheException("Failed to get node for cache: " + cacheName); - rcvrMsg = allow ? new DataStreamerReceiverMessage(DataStreamerCacheUpdaters.individual()) : null; + rcvrMsg = new DataStreamerReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); } /** {@inheritDoc} */ @@ -1999,11 +1997,15 @@ private void submit( DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; + // The updaters the streamer ships with are named rather than sent: every node has them. + DataStreamerBuiltInUpdater builtIn = DataStreamerBuiltInUpdater.of(rcvrMsg0.receiver()); + DataStreamerRequest req = new DataStreamerRequest( reqId, topicId, cacheName, - rcvrMsg0, + builtIn == null ? rcvrMsg0 : null, + builtIn, entries, true, skipStore, @@ -2012,7 +2014,7 @@ private void submit( dep != null ? jobPda0.deployClass().getName() : null, dep == null, topVer, - rcvrMsg0 == null ? partId : NO_STRIPE); + builtIn == DataStreamerBuiltInUpdater.ISOLATED ? partId : NO_STRIPE); try { ctx.io().sendToGridTopic(node, TOPIC_DATASTREAM, req, plc); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 0c8341923d312..2007d282d6eaa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -49,7 +49,7 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache updater; {@code null} for the default one. */ + /** Cache updater; {@code null} when {@link #builtInUpdater} names it. */ @GridToStringExclude @Order(3) DataStreamerReceiverMessage updaterMsg; @@ -90,6 +90,10 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(12) int partId; + /** Cache updater of the streamer itself; {@code null} when {@link #updaterMsg} carries a user one. */ + @Order(13) + DataStreamerBuiltInUpdater builtInUpdater; + /** Empty constructor. */ public DataStreamerRequest() { // No-op. @@ -99,7 +103,8 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterMsg Cache updater, {@code null} for the default one. + * @param updaterMsg Cache updater, {@code null} when {@code builtInUpdater} names it. + * @param builtInUpdater Cache updater of the streamer itself, {@code null} for a user one. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -115,6 +120,7 @@ public DataStreamerRequest( IgniteUuid resTopicId, @Nullable String cacheName, @Nullable DataStreamerReceiverMessage updaterMsg, + @Nullable DataStreamerBuiltInUpdater builtInUpdater, Collection entries, boolean ignoreDepOwnership, boolean skipStore, @@ -131,6 +137,7 @@ public DataStreamerRequest( this.resTopicId = resTopicId; this.cacheName = cacheName; this.updaterMsg = updaterMsg; + this.builtInUpdater = builtInUpdater; this.entries = entries; this.ignoreDepOwnership = ignoreDepOwnership; this.skipStore = skipStore; @@ -157,9 +164,9 @@ String cacheName() { return cacheName; } - /** @return Updater, {@code null} for the default one. */ - @Nullable StreamReceiver updater() { - return updaterMsg != null ? updaterMsg.receiver() : null; + /** @return Updater: the one carried by the request, or the streamer's own that it named. */ + StreamReceiver updater() { + return updaterMsg != null ? updaterMsg.receiver() : builtInUpdater.updater(); } /** @return Entries to update. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 46d63088a5143..395c1d65bc4cf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -182,7 +182,7 @@ private void streamToRemoteNode(@Nullable StreamReceiver rcvr) t */ @Test public void testReceiverMarshalledOncePerStreamer() throws Exception { - streamToRemoteNode(DataStreamerCacheUpdaters.batched()); + streamToRemoteNode(new TestReceiver()); assertTrue("Expected more than one request to a remote node, got " + sentReceivers.size(), sentReceivers.size() > 1); @@ -196,18 +196,18 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { } /** - * The isolated updater ships with every node, so requests carry no updater at all and the data still lands. + * The updaters the streamer ships with are named rather than sent, and the data still lands. * * @throws Exception If failed. */ @Test - public void testIsolatedUpdaterIsNotSent() throws Exception { + public void testBuiltInUpdaterIsNotSent() throws Exception { streamToRemoteNode(null); assertTrue("Expected requests to a remote node, got " + sentReceivers.size(), !sentReceivers.isEmpty()); for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertNull("The isolated updater was sent with a request", rcvr); + assertNull("A built-in updater was sent with a request", rcvr); IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); @@ -215,6 +215,21 @@ public void testIsolatedUpdaterIsNotSent() throws Exception { assertEquals(i, cache.get(i)); } + /** + * A built-in receiver set explicitly is named rather than sent, just like the default one. + * + * @throws Exception If failed. + */ + @Test + public void testBuiltInReceiverIsNotSent() throws Exception { + streamToRemoteNode(DataStreamerCacheUpdaters.batched()); + + assertFalse("Expected requests to a remote node", sentReceivers.isEmpty()); + + for (DataStreamerReceiverMessage rcvr : sentReceivers) + assertNull("A built-in updater was sent with a request", rcvr); + } + /** * Test inconsistency log warning of the streamer. Default receiver goes first and is set again after a consistent * receiver. The warning must appear only once. @@ -737,6 +752,18 @@ private CacheConfiguration cacheConfiguration() { return cacheCfg; } + /** A receiver of the test itself: unlike the built-in ones, it travels with the requests. */ + private static class TestReceiver implements StreamReceiver { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override public void receive(IgniteCache cache, Collection> entries) { + for (Map.Entry e : entries) + cache.put(e.getKey(), e.getValue()); + } + } + /** * Simulate stale (not up-to-date) topology */ @@ -772,6 +799,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { req.resTopicId, req.cacheName(), req.updaterMsg, + req.builtInUpdater, req.entries(), req.ignoreDeploymentOwnership(), req.skipStore(), From 34688f973f756c0cafb18d828d85bd6fb96fafc8 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 18:38:00 +0300 Subject: [PATCH 21/35] IGNITE-27977 Let the request decide how to carry its updater The caller was choosing between a copy and a name and explaining that choice in a comment. The request now takes the receiver and stores it the way it travels best: as a name when the streamer ships with it, as a serialized copy otherwise. The comment and the invariant assert went away with the choice. Also narrowed DataStreamerBuiltInUpdater to package-private and named the check the processor makes on the request. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamProcessor.java | 3 +-- .../datastreamer/DataStreamerBuiltInUpdater.java | 2 +- .../processors/datastreamer/DataStreamerImpl.java | 8 ++------ .../datastreamer/DataStreamerRequest.java | 15 +++++++++------ .../datastreamer/DataStreamerImplSelfTest.java | 3 +-- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index d83c630f38595..4ca1642b5e06f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -234,8 +234,7 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { updater = req.updater(); - // Only a user updater arrives with the request and needs its resources injected. - if (req.updaterMsg != null) + if (req.hasUserUpdater()) ctx.resource().injectGeneric(updater); } catch (IgniteCheckedException e) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java index 406ac85a9b2d6..5a75f050e0bb4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -24,7 +24,7 @@ * The updaters DataStreamer ships with. Every node has them, so a request names the one it needs instead of carrying * a serialized copy. */ -public enum DataStreamerBuiltInUpdater { +enum DataStreamerBuiltInUpdater { /** {@link DataStreamerImpl#ISOLATED_UPDATER}. */ ISOLATED(DataStreamerImpl.ISOLATED_UPDATER), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 8200b70eaeae1..cd1570e19b98f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -1997,15 +1997,11 @@ private void submit( DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; - // The updaters the streamer ships with are named rather than sent: every node has them. - DataStreamerBuiltInUpdater builtIn = DataStreamerBuiltInUpdater.of(rcvrMsg0.receiver()); - DataStreamerRequest req = new DataStreamerRequest( reqId, topicId, cacheName, - builtIn == null ? rcvrMsg0 : null, - builtIn, + rcvrMsg0, entries, true, skipStore, @@ -2014,7 +2010,7 @@ private void submit( dep != null ? jobPda0.deployClass().getName() : null, dep == null, topVer, - builtIn == DataStreamerBuiltInUpdater.ISOLATED ? partId : NO_STRIPE); + rcvrMsg0.receiver() == ISOLATED_UPDATER ? partId : NO_STRIPE); try { ctx.io().sendToGridTopic(node, TOPIC_DATASTREAM, req, plc); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 2007d282d6eaa..7cea7f49da5dc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -103,8 +103,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterMsg Cache updater, {@code null} when {@code builtInUpdater} names it. - * @param builtInUpdater Cache updater of the streamer itself, {@code null} for a user one. + * @param updaterMsg Cache updater. One of the streamer's own is stored as a name, see {@link #builtInUpdater}. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -119,8 +118,7 @@ public DataStreamerRequest( long reqId, IgniteUuid resTopicId, @Nullable String cacheName, - @Nullable DataStreamerReceiverMessage updaterMsg, - @Nullable DataStreamerBuiltInUpdater builtInUpdater, + DataStreamerReceiverMessage updaterMsg, Collection entries, boolean ignoreDepOwnership, boolean skipStore, @@ -136,8 +134,8 @@ public DataStreamerRequest( this.reqId = reqId; this.resTopicId = resTopicId; this.cacheName = cacheName; - this.updaterMsg = updaterMsg; - this.builtInUpdater = builtInUpdater; + builtInUpdater = DataStreamerBuiltInUpdater.of(updaterMsg.receiver()); + this.updaterMsg = builtInUpdater == null ? updaterMsg : null; this.entries = entries; this.ignoreDepOwnership = ignoreDepOwnership; this.skipStore = skipStore; @@ -164,6 +162,11 @@ String cacheName() { return cacheName; } + /** @return {@code True} if the updater travelled with the request instead of being named. */ + boolean hasUserUpdater() { + return updaterMsg != null; + } + /** @return Updater: the one carried by the request, or the streamer's own that it named. */ StreamReceiver updater() { return updaterMsg != null ? updaterMsg.receiver() : builtInUpdater.updater(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 395c1d65bc4cf..cb10ec3a05376 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -798,8 +798,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { req.requestId(), req.resTopicId, req.cacheName(), - req.updaterMsg, - req.builtInUpdater, + new DataStreamerReceiverMessage(req.updater()), req.entries(), req.ignoreDeploymentOwnership(), req.skipStore(), From 717f5dab7c3842b649decc7896e2c635bd2a237d Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 18:48:17 +0300 Subject: [PATCH 22/35] IGNITE-27977 Make allowOverwrite the receiver choice it is The flag was a third way of saying which receiver is in use: the getter compared against ISOLATED_UPDATER through a private helper, the setter built the carrier by hand, and the warning site asked the helper. All three now go through receiver(), so the state has a single owner and the flag reads as what it means - "the receiver is not the Isolated one". Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerImpl.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index cd1570e19b98f..63293948b3a3b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -495,14 +495,9 @@ private StreamReceiver receiver() { return (StreamReceiver)rcvrMsg.receiver(); } - /** @return {@code True} if the default, Isolated receiver is in use. */ - private boolean isolated() { - return rcvrMsg.receiver() == ISOLATED_UPDATER; - } - /** {@inheritDoc} */ @Override public boolean allowOverwrite() { - return !isolated(); + return receiver() != ISOLATED_UPDATER; } /** {@inheritDoc} */ @@ -515,7 +510,7 @@ private boolean isolated() { if (node == null) throw new CacheException("Failed to get node for cache: " + cacheName); - rcvrMsg = new DataStreamerReceiverMessage(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); + receiver(allow ? DataStreamerCacheUpdaters.individual() : ISOLATED_UPDATER); } /** {@inheritDoc} */ @@ -663,7 +658,7 @@ public IgniteFuture addDataInternal(Collection e lock(false); - if (isolated() && inconsistencyWarned.compareAndSet(false, true)) + if (!allowOverwrite() && inconsistencyWarned.compareAndSet(false, true)) log.warning(WRN_INCONSISTENT_UPDATES); try { From c619b7c2a8c7560cef622a1e0311ef544eaafd6d Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 18:56:48 +0300 Subject: [PATCH 23/35] IGNITE-27977 Build a receiver message only for a user receiver The updaters the streamer ships with never change, so their messages are built once, with the enum constant that names them. Setting a receiver picks the matching one, and only a user receiver gets a message of its own - switching allowOverwrite back and forth allocates nothing, and the initial state is one of the constants rather than a special case. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamerBuiltInUpdater.java | 10 ++++++++++ .../processors/datastreamer/DataStreamerImpl.java | 8 +++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java index 5a75f050e0bb4..8ac48ab43a915 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -40,9 +40,14 @@ enum DataStreamerBuiltInUpdater { /** */ private final StreamReceiver updater; + /** */ + private final DataStreamerReceiverMessage msg; + /** @param updater Updater this constant stands for. */ DataStreamerBuiltInUpdater(StreamReceiver updater) { this.updater = updater; + + msg = new DataStreamerReceiverMessage(updater); } /** @return Updater of this node. */ @@ -50,6 +55,11 @@ enum DataStreamerBuiltInUpdater { return updater; } + /** @return Message holding {@link #updater()}; one per constant, as these updaters never change. */ + DataStreamerReceiverMessage message() { + return msg; + } + /** * @param updater Updater to look up. * @return Constant standing for {@code updater}, or {@code null} when it is not one the streamer ships with. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 63293948b3a3b..813e7982d7a2d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -152,8 +152,8 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; - /** Cache receiver in its message. */ - private volatile DataStreamerReceiverMessage rcvrMsg = new DataStreamerReceiverMessage(ISOLATED_UPDATER); + /** Cache receiver in the message that carries it. */ + private volatile DataStreamerReceiverMessage rcvrMsg = DataStreamerBuiltInUpdater.ISOLATED.message(); /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -486,7 +486,9 @@ public IgniteInternalFuture internalFuture() { @Override public void receiver(StreamReceiver rcvr) { A.notNull(rcvr, "rcvr"); - rcvrMsg = new DataStreamerReceiverMessage(rcvr); + DataStreamerBuiltInUpdater builtIn = DataStreamerBuiltInUpdater.of(rcvr); + + rcvrMsg = builtIn != null ? builtIn.message() : new DataStreamerReceiverMessage(rcvr); } /** @return Cache receiver. */ From 51f1ed8c24e0060a2ca0f1b1f6de267c688301d2 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 19:09:34 +0300 Subject: [PATCH 24/35] IGNITE-27977 Carry both forms in the receiver message, as GridTopicMessage does The request held the carrier and the name in two fields; now the carrier holds both, the way GridTopicMessage keeps either a topic ordinal or a serialized topic. The request is back to one field, and asking "did the receiver travel or was it named" is a question to the carrier. Co-Authored-By: Claude Opus 5 --- .../DataStreamerBuiltInUpdater.java | 2 +- .../DataStreamerReceiverMessage.java | 22 +++++++++++++++---- .../datastreamer/DataStreamerRequest.java | 17 +++++--------- .../DataStreamerImplSelfTest.java | 8 +++---- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java index 8ac48ab43a915..21029587ca202 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -47,7 +47,7 @@ enum DataStreamerBuiltInUpdater { DataStreamerBuiltInUpdater(StreamReceiver updater) { this.updater = updater; - msg = new DataStreamerReceiverMessage(updater); + msg = new DataStreamerReceiverMessage(this); } /** @return Updater of this node. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index e1c36cf17cc43..b374252189ec4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -26,7 +26,7 @@ /** DataStreamer cache receiver/updater message. */ @UseBinaryMarshaller public class DataStreamerReceiverMessage implements Message { - /** DataStreamer cache receiver/updater. */ + /** DataStreamer cache receiver/updater; {@code null} when {@link #builtIn} names it. */ @Marshalled("rcvrBytes") StreamReceiver rcvr; @@ -34,18 +34,32 @@ public class DataStreamerReceiverMessage implements Message { @Order(0) volatile byte[] rcvrBytes; + /** The updater every node has; {@code null} when {@link #rcvr} is a user one. */ + @Order(1) + DataStreamerBuiltInUpdater builtIn; + /** Empty constructor for serialization purposes. */ public DataStreamerReceiverMessage() { // No-op. } - /** @param rcvr Receiver. */ + /** @param rcvr User receiver. */ DataStreamerReceiverMessage(StreamReceiver rcvr) { this.rcvr = rcvr; } - /** @return Receiver. */ + /** @param builtIn Updater every node has, named rather than carried. */ + DataStreamerReceiverMessage(DataStreamerBuiltInUpdater builtIn) { + this.builtIn = builtIn; + } + + /** @return {@code True} if the receiver travels with this message rather than being named. */ + boolean user() { + return builtIn == null; + } + + /** @return Receiver: the one carried here, or the node's own that this message names. */ StreamReceiver receiver() { - return rcvr; + return builtIn == null ? rcvr : builtIn.updater(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 7cea7f49da5dc..3ef9f89ffdb54 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -49,7 +49,7 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(2) String cacheName; - /** Cache updater; {@code null} when {@link #builtInUpdater} names it. */ + /** Cache updater. */ @GridToStringExclude @Order(3) DataStreamerReceiverMessage updaterMsg; @@ -90,10 +90,6 @@ public class DataStreamerRequest implements DeferredUnmarshalMessage, CacheIdAwa @Order(12) int partId; - /** Cache updater of the streamer itself; {@code null} when {@link #updaterMsg} carries a user one. */ - @Order(13) - DataStreamerBuiltInUpdater builtInUpdater; - /** Empty constructor. */ public DataStreamerRequest() { // No-op. @@ -103,7 +99,7 @@ public DataStreamerRequest() { * @param reqId Request ID. * @param resTopicId Response topic ID. * @param cacheName Cache name. - * @param updaterMsg Cache updater. One of the streamer's own is stored as a name, see {@link #builtInUpdater}. + * @param updaterMsg Cache updater. * @param entries Entries to put. * @param ignoreDepOwnership Ignore ownership. * @param skipStore Skip store flag. @@ -134,8 +130,7 @@ public DataStreamerRequest( this.reqId = reqId; this.resTopicId = resTopicId; this.cacheName = cacheName; - builtInUpdater = DataStreamerBuiltInUpdater.of(updaterMsg.receiver()); - this.updaterMsg = builtInUpdater == null ? updaterMsg : null; + this.updaterMsg = updaterMsg; this.entries = entries; this.ignoreDepOwnership = ignoreDepOwnership; this.skipStore = skipStore; @@ -164,12 +159,12 @@ String cacheName() { /** @return {@code True} if the updater travelled with the request instead of being named. */ boolean hasUserUpdater() { - return updaterMsg != null; + return updaterMsg.user(); } - /** @return Updater: the one carried by the request, or the streamer's own that it named. */ + /** @return Updater. */ StreamReceiver updater() { - return updaterMsg != null ? updaterMsg.receiver() : builtInUpdater.updater(); + return updaterMsg.receiver(); } /** @return Entries to update. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index cb10ec3a05376..572cb826d7f55 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -90,7 +90,7 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Indicates whether we need to make the topology stale */ private static boolean needStaleTop = false; - /** Receiver carriers of the streamer requests sent since the current test started; {@code null} for the isolated one. */ + /** Receiver carriers of the streamer requests sent since the current test started. */ private static final List sentReceivers = Collections.synchronizedList(new ArrayList<>()); /** {@inheritDoc} */ @@ -207,7 +207,7 @@ public void testBuiltInUpdaterIsNotSent() throws Exception { assertTrue("Expected requests to a remote node, got " + sentReceivers.size(), !sentReceivers.isEmpty()); for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertNull("A built-in updater was sent with a request", rcvr); + assertFalse("A built-in updater was sent with a request", rcvr.user()); IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); @@ -227,7 +227,7 @@ public void testBuiltInReceiverIsNotSent() throws Exception { assertFalse("Expected requests to a remote node", sentReceivers.isEmpty()); for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertNull("A built-in updater was sent with a request", rcvr); + assertFalse("A built-in updater was sent with a request", rcvr.user()); } /** @@ -798,7 +798,7 @@ private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { req.requestId(), req.resTopicId, req.cacheName(), - new DataStreamerReceiverMessage(req.updater()), + req.updaterMsg, req.entries(), req.ignoreDeploymentOwnership(), req.skipStore(), From abc9a829a756bd99ea4c6fbd3c6a867c0b061f26 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 19:15:24 +0300 Subject: [PATCH 25/35] IGNITE-27977 Decide the receiver by logic, not by an initial assignment The field no longer starts out holding the Isolated updater. It is empty until someone sets a receiver, and the rule "nothing set means Isolated" lives in one place, where the receiver is read. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerImpl.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 813e7982d7a2d..b7bd431c302f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -152,8 +152,8 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; - /** Cache receiver in the message that carries it. */ - private volatile DataStreamerReceiverMessage rcvrMsg = DataStreamerBuiltInUpdater.ISOLATED.message(); + /** Cache receiver in the message that carries it; {@code null} while none is set. */ + private volatile DataStreamerReceiverMessage rcvrMsg; /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -491,10 +491,17 @@ public IgniteInternalFuture internalFuture() { rcvrMsg = builtIn != null ? builtIn.message() : new DataStreamerReceiverMessage(rcvr); } + /** @return Message of the receiver in use, the Isolated updater until one is set. */ + private DataStreamerReceiverMessage receiverMessage() { + DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; + + return rcvrMsg0 != null ? rcvrMsg0 : DataStreamerBuiltInUpdater.ISOLATED.message(); + } + /** @return Cache receiver. */ @SuppressWarnings("unchecked") private StreamReceiver receiver() { - return (StreamReceiver)rcvrMsg.receiver(); + return (StreamReceiver)receiverMessage().receiver(); } /** {@inheritDoc} */ @@ -1992,7 +1999,7 @@ private void submit( if (topVer == null) topVer = ctx.cache().context().exchange().readyAffinityVersion(); - DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; + DataStreamerReceiverMessage rcvrMsg0 = receiverMessage(); DataStreamerRequest req = new DataStreamerRequest( reqId, From 8d945565703fba194af60cb164a46eda9ee4ac9c Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 19:29:23 +0300 Subject: [PATCH 26/35] IGNITE-27977 Assert that a built-in updater is never carried The carrier has two constructors and the invariant between them lived only in the caller. Now the user one states it, and a receiver that should have travelled by name fails where the mistake is made rather than silently costing a serialization per batch. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerReceiverMessage.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index b374252189ec4..329e143f0a70e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -45,6 +45,8 @@ public DataStreamerReceiverMessage() { /** @param rcvr User receiver. */ DataStreamerReceiverMessage(StreamReceiver rcvr) { + assert DataStreamerBuiltInUpdater.of(rcvr) == null : "A built-in updater travels by name: " + rcvr; + this.rcvr = rcvr; } From 67a0ba74e5a4cd3d2dc150978923115352b74191 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 20:11:44 +0300 Subject: [PATCH 27/35] IGNITE-27977 Review: built-in on demand, and built-in/custom notation Both the updater a constant stands for and its message are built on demand now: a streamer that keeps the default receiver never loads DataStreamerCacheUpdaters, and a node that does not stream builds neither. Wording follows the review: "built-in" for the updaters every node has - only the Isolated one belongs to the streamer, the rest are external to it - and "custom" for the user's, so user() became custom() and hasUserUpdater() became hasCustomUpdater(). Both nullable fields of the carrier are marked, and the warning site says which updater it is about. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamProcessor.java | 2 +- .../DataStreamerBuiltInUpdater.java | 39 ++++++++++++------- .../datastreamer/DataStreamerImpl.java | 1 + .../DataStreamerReceiverMessage.java | 17 ++++---- .../datastreamer/DataStreamerRequest.java | 6 +-- .../DataStreamerImplSelfTest.java | 8 ++-- 6 files changed, 43 insertions(+), 30 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index 4ca1642b5e06f..668fcaa53e6f9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -234,7 +234,7 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { updater = req.updater(); - if (req.hasUserUpdater()) + if (req.hasCustomUpdater()) ctx.resource().injectGeneric(updater); } catch (IgniteCheckedException e) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java index 21029587ca202..f1887fc6b1cac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -17,56 +17,65 @@ package org.apache.ignite.internal.processors.datastreamer; +import java.util.function.Supplier; import org.apache.ignite.stream.StreamReceiver; import org.jetbrains.annotations.Nullable; /** - * The updaters DataStreamer ships with. Every node has them, so a request names the one it needs instead of carrying - * a serialized copy. + * The built-in updaters. Every node has them, so a request names the one it needs instead of carrying a serialized + * copy. Both the updater and its message are built on demand: a node that streams with a custom receiver, or does not + * stream at all, builds neither. */ enum DataStreamerBuiltInUpdater { /** {@link DataStreamerImpl#ISOLATED_UPDATER}. */ - ISOLATED(DataStreamerImpl.ISOLATED_UPDATER), + ISOLATED(() -> DataStreamerImpl.ISOLATED_UPDATER), /** {@link DataStreamerCacheUpdaters#individual()}. */ - INDIVIDUAL(DataStreamerCacheUpdaters.individual()), + INDIVIDUAL(DataStreamerCacheUpdaters::individual), /** {@link DataStreamerCacheUpdaters#batched()}. */ - BATCHED(DataStreamerCacheUpdaters.batched()), + BATCHED(DataStreamerCacheUpdaters::batched), /** {@link DataStreamerCacheUpdaters#batchedSorted()}. */ - BATCHED_SORTED(DataStreamerCacheUpdaters.batchedSorted()); + BATCHED_SORTED(DataStreamerCacheUpdaters::batchedSorted); /** */ - private final StreamReceiver updater; + private final Supplier> updaterSupplier; /** */ - private final DataStreamerReceiverMessage msg; + private StreamReceiver updater; - /** @param updater Updater this constant stands for. */ - DataStreamerBuiltInUpdater(StreamReceiver updater) { - this.updater = updater; + /** */ + private DataStreamerReceiverMessage msg; - msg = new DataStreamerReceiverMessage(this); + /** @param updaterSupplier Supplier of the updater this constant stands for. */ + DataStreamerBuiltInUpdater(Supplier> updaterSupplier) { + this.updaterSupplier = updaterSupplier; } /** @return Updater of this node. */ StreamReceiver updater() { + if (updater == null) + updater = updaterSupplier.get(); + return updater; } - /** @return Message holding {@link #updater()}; one per constant, as these updaters never change. */ + /** @return Message naming this updater; one per constant, as a built-in updater never changes. */ DataStreamerReceiverMessage message() { + if (msg == null) + msg = new DataStreamerReceiverMessage(this); + return msg; } /** * @param updater Updater to look up. - * @return Constant standing for {@code updater}, or {@code null} when it is not one the streamer ships with. + * @return Constant standing for {@code updater}, or {@code null} when it is a custom one. */ static @Nullable DataStreamerBuiltInUpdater of(StreamReceiver updater) { for (DataStreamerBuiltInUpdater builtIn : values()) { - if (builtIn.updater == updater) + if (builtIn.updater() == updater) return builtIn; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index b7bd431c302f5..7c63cc3578054 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -667,6 +667,7 @@ public IgniteFuture addDataInternal(Collection e lock(false); + // Without overwrite the Isolated updater is in use, and it is the one the warning is about. if (!allowOverwrite() && inconsistencyWarned.compareAndSet(false, true)) log.warning(WRN_INCONSISTENT_UPDATES); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index 329e143f0a70e..a264b0cf0fdb1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -22,11 +22,13 @@ import org.apache.ignite.internal.UseBinaryMarshaller; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.stream.StreamReceiver; +import org.jetbrains.annotations.Nullable; /** DataStreamer cache receiver/updater message. */ @UseBinaryMarshaller public class DataStreamerReceiverMessage implements Message { - /** DataStreamer cache receiver/updater; {@code null} when {@link #builtIn} names it. */ + /** Custom cache receiver/updater; {@code null} when {@link #builtIn} is effective. */ + @Nullable @Marshalled("rcvrBytes") StreamReceiver rcvr; @@ -34,7 +36,8 @@ public class DataStreamerReceiverMessage implements Message { @Order(0) volatile byte[] rcvrBytes; - /** The updater every node has; {@code null} when {@link #rcvr} is a user one. */ + /** A built-in updater every node has; {@code null} when {@link #rcvr} is effective. */ + @Nullable @Order(1) DataStreamerBuiltInUpdater builtIn; @@ -43,24 +46,24 @@ public DataStreamerReceiverMessage() { // No-op. } - /** @param rcvr User receiver. */ + /** @param rcvr Custom receiver. */ DataStreamerReceiverMessage(StreamReceiver rcvr) { assert DataStreamerBuiltInUpdater.of(rcvr) == null : "A built-in updater travels by name: " + rcvr; this.rcvr = rcvr; } - /** @param builtIn Updater every node has, named rather than carried. */ + /** @param builtIn A built-in updater every node has, named rather than carried. */ DataStreamerReceiverMessage(DataStreamerBuiltInUpdater builtIn) { this.builtIn = builtIn; } - /** @return {@code True} if the receiver travels with this message rather than being named. */ - boolean user() { + /** @return {@code True} if this is a custom receiver, {@code false} if a built-in one. */ + boolean custom() { return builtIn == null; } - /** @return Receiver: the one carried here, or the node's own that this message names. */ + /** @return Receiver: the custom one carried here, or the built-in one this message names. */ StreamReceiver receiver() { return builtIn == null ? rcvr : builtIn.updater(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 3ef9f89ffdb54..14125cee9c675 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -157,9 +157,9 @@ String cacheName() { return cacheName; } - /** @return {@code True} if the updater travelled with the request instead of being named. */ - boolean hasUserUpdater() { - return updaterMsg.user(); + /** @return {@code True} if the request carries a custom updater rather than naming a built-in one. */ + boolean hasCustomUpdater() { + return updaterMsg.custom(); } /** @return Updater. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 572cb826d7f55..3bf29e5ef3761 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -196,7 +196,7 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { } /** - * The updaters the streamer ships with are named rather than sent, and the data still lands. + * A built-in updater is named rather than sent, and the data still lands. * * @throws Exception If failed. */ @@ -207,7 +207,7 @@ public void testBuiltInUpdaterIsNotSent() throws Exception { assertTrue("Expected requests to a remote node, got " + sentReceivers.size(), !sentReceivers.isEmpty()); for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertFalse("A built-in updater was sent with a request", rcvr.user()); + assertFalse("A built-in updater was sent with a request", rcvr.custom()); IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); @@ -227,7 +227,7 @@ public void testBuiltInReceiverIsNotSent() throws Exception { assertFalse("Expected requests to a remote node", sentReceivers.isEmpty()); for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertFalse("A built-in updater was sent with a request", rcvr.user()); + assertFalse("A built-in updater was sent with a request", rcvr.custom()); } /** @@ -752,7 +752,7 @@ private CacheConfiguration cacheConfiguration() { return cacheCfg; } - /** A receiver of the test itself: unlike the built-in ones, it travels with the requests. */ + /** A custom receiver: unlike the built-in ones, it travels with the requests. */ private static class TestReceiver implements StreamReceiver { /** */ private static final long serialVersionUID = 0L; From 66ee960165e895c8faf9049f8083d1b0e922e05e Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 20:27:54 +0300 Subject: [PATCH 28/35] IGNITE-27977 Review: one test over every built-in updater The two built-in tests differed only in how the receiver was set, so they became one loop over DataStreamerBuiltInUpdater.values(). What the SPI collects is now the set of distinct updaters that left the node - the serialized bytes, or the constant - so a test states its expectation as one equality instead of walking a list. The collection is a test instance field, the helper moved below the tests that use it and says why it waits for the partition map, and it is named startGridsAndStream, since that is what it does. Co-Authored-By: Claude Opus 5 --- .../DataStreamerImplSelfTest.java | 100 ++++++++---------- 1 file changed, 43 insertions(+), 57 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 3bf29e5ef3761..ea6f3a45d0a33 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -21,9 +21,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -61,7 +63,6 @@ import org.apache.ignite.testframework.LogListener; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.logging.log4j.core.appender.WriterAppender; -import org.jetbrains.annotations.Nullable; import org.junit.Ignore; import org.junit.Test; @@ -90,8 +91,8 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Indicates whether we need to make the topology stale */ private static boolean needStaleTop = false; - /** Receiver carriers of the streamer requests sent since the current test started. */ - private static final List sentReceivers = Collections.synchronizedList(new ArrayList<>()); + /** Distinct updaters sent since the current test started: the serialized bytes, or the built-in constant. */ + private final Set sentUpdaters = Collections.synchronizedSet(new HashSet<>()); /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { @@ -147,33 +148,6 @@ public void testCloseWithCancellation() throws Exception { assertTrue(fut.isDone()); } - /** - * Streams {@link #KEYS_COUNT} entries from the first node, one entry per request, and collects the receiver - * carrier of every request that leaves the node. - * - * @param rcvr Receiver to set, or {@code null} to keep the default one. - * @throws Exception If failed. - */ - private void streamToRemoteNode(@Nullable StreamReceiver rcvr) throws Exception { - cnt = 0; - - startGrids(2); - - awaitPartitionMapExchange(); - - sentReceivers.clear(); - - try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { - if (rcvr != null) - ldr.receiver(rcvr); - - ldr.perNodeBufferSize(1); - - for (int i = 0; i < KEYS_COUNT; i++) - ldr.addData(i, i); - } - } - /** * The receiver does not change between batches, so it is marshalled once: every request carries the very bytes * produced for the first one. @@ -182,52 +156,61 @@ private void streamToRemoteNode(@Nullable StreamReceiver rcvr) t */ @Test public void testReceiverMarshalledOncePerStreamer() throws Exception { - streamToRemoteNode(new TestReceiver()); - - assertTrue("Expected more than one request to a remote node, got " + sentReceivers.size(), - sentReceivers.size() > 1); + startGridsAndStream(new TestReceiver()); - DataStreamerReceiverMessage first = F.first(sentReceivers); + assertEquals("The receiver was marshalled more than once", 1, sentUpdaters.size()); - assertNotNull(first.rcvrBytes); - - for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertTrue("The receiver was marshalled more than once", first.rcvrBytes == rcvr.rcvrBytes); + assertTrue("The receiver was named instead of sent", F.first(sentUpdaters) instanceof byte[]); } /** - * A built-in updater is named rather than sent, and the data still lands. + * Every built-in updater is named rather than sent, and the data still lands. * * @throws Exception If failed. */ @Test public void testBuiltInUpdaterIsNotSent() throws Exception { - streamToRemoteNode(null); + for (DataStreamerBuiltInUpdater builtIn : DataStreamerBuiltInUpdater.values()) { + startGridsAndStream(builtIn.updater()); - assertTrue("Expected requests to a remote node, got " + sentReceivers.size(), !sentReceivers.isEmpty()); + assertEquals("Expected " + builtIn + " to be named, not sent", Collections.singleton(builtIn), + sentUpdaters); - for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertFalse("A built-in updater was sent with a request", rcvr.custom()); + IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); - IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); + for (int i = 0; i < KEYS_COUNT; i++) + assertEquals(i, cache.get(i)); - for (int i = 0; i < KEYS_COUNT; i++) - assertEquals(i, cache.get(i)); + stopAllGrids(); + } } /** - * A built-in receiver set explicitly is named rather than sent, just like the default one. + * Starts two nodes and streams {@link #KEYS_COUNT} entries from the first one, a request per entry, collecting + * the updaters they carry. Waits for the partition map first: until it is ready every partition is primary here, + * and a streamer that overwrites sends nothing to the remote node. * + * @param rcvr Receiver to stream with. * @throws Exception If failed. */ - @Test - public void testBuiltInReceiverIsNotSent() throws Exception { - streamToRemoteNode(DataStreamerCacheUpdaters.batched()); + @SuppressWarnings("unchecked") + private void startGridsAndStream(StreamReceiver rcvr) throws Exception { + cnt = 0; + + startGrids(2); + + awaitPartitionMapExchange(); + + sentUpdaters.clear(); - assertFalse("Expected requests to a remote node", sentReceivers.isEmpty()); + try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + ldr.receiver((StreamReceiver)rcvr); + + ldr.perNodeBufferSize(1); - for (DataStreamerReceiverMessage rcvr : sentReceivers) - assertFalse("A built-in updater was sent with a request", rcvr.custom()); + for (int i = 0; i < KEYS_COUNT; i++) + ldr.addData(i, i); + } } /** @@ -767,14 +750,17 @@ private static class TestReceiver implements StreamReceiver { /** * Simulate stale (not up-to-date) topology */ - private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { + private class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { Message sentMsg = msg instanceof GridIoMessage ? ((GridIoMessage)msg).message() : null; // Already marshalled at this point. - if (sentMsg instanceof DataStreamerRequest) - sentReceivers.add(((DataStreamerRequest)sentMsg).updaterMsg); + if (sentMsg instanceof DataStreamerRequest) { + DataStreamerReceiverMessage updaterMsg = ((DataStreamerRequest)sentMsg).updaterMsg; + + sentUpdaters.add(updaterMsg.custom() ? updaterMsg.rcvrBytes : updaterMsg.builtIn); + } // Send stale topology only in the first request to avoid indefinitely getting failures. if (needStaleTop) { From 3a4d13ad7a0242466ec6a1c1d50568c9ade0a7f5 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 20:30:56 +0300 Subject: [PATCH 29/35] IGNITE-27977 Review: keep the updaters eager, only the messages lazy The updaters are singletons that already exist, so resolving them through a supplier bought nothing and cost a level of indirection. Reverted to holding them directly; the message naming a constant is still built on first use. Co-Authored-By: Claude Opus 5 --- .../DataStreamerBuiltInUpdater.java | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java index f1887fc6b1cac..85d2c869afc63 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -17,47 +17,39 @@ package org.apache.ignite.internal.processors.datastreamer; -import java.util.function.Supplier; import org.apache.ignite.stream.StreamReceiver; import org.jetbrains.annotations.Nullable; /** * The built-in updaters. Every node has them, so a request names the one it needs instead of carrying a serialized - * copy. Both the updater and its message are built on demand: a node that streams with a custom receiver, or does not - * stream at all, builds neither. + * copy. The message naming one is built on demand: a node that streams with a custom receiver builds none. */ enum DataStreamerBuiltInUpdater { /** {@link DataStreamerImpl#ISOLATED_UPDATER}. */ - ISOLATED(() -> DataStreamerImpl.ISOLATED_UPDATER), + ISOLATED(DataStreamerImpl.ISOLATED_UPDATER), /** {@link DataStreamerCacheUpdaters#individual()}. */ - INDIVIDUAL(DataStreamerCacheUpdaters::individual), + INDIVIDUAL(DataStreamerCacheUpdaters.individual()), /** {@link DataStreamerCacheUpdaters#batched()}. */ - BATCHED(DataStreamerCacheUpdaters::batched), + BATCHED(DataStreamerCacheUpdaters.batched()), /** {@link DataStreamerCacheUpdaters#batchedSorted()}. */ - BATCHED_SORTED(DataStreamerCacheUpdaters::batchedSorted); + BATCHED_SORTED(DataStreamerCacheUpdaters.batchedSorted()); /** */ - private final Supplier> updaterSupplier; - - /** */ - private StreamReceiver updater; + private final StreamReceiver updater; /** */ private DataStreamerReceiverMessage msg; - /** @param updaterSupplier Supplier of the updater this constant stands for. */ - DataStreamerBuiltInUpdater(Supplier> updaterSupplier) { - this.updaterSupplier = updaterSupplier; + /** @param updater Updater this constant stands for. */ + DataStreamerBuiltInUpdater(StreamReceiver updater) { + this.updater = updater; } /** @return Updater of this node. */ StreamReceiver updater() { - if (updater == null) - updater = updaterSupplier.get(); - return updater; } @@ -75,7 +67,7 @@ DataStreamerReceiverMessage message() { */ static @Nullable DataStreamerBuiltInUpdater of(StreamReceiver updater) { for (DataStreamerBuiltInUpdater builtIn : values()) { - if (builtIn.updater() == updater) + if (builtIn.updater == updater) return builtIn; } From 985216b589d481bff2030972d07cf1aaa25c413f Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 20:38:51 +0300 Subject: [PATCH 30/35] IGNITE-27977 Review: nullable fields, one notation, Ignite-style getter @Nullable now sits on the field declarations, including the bytes; the javadocs say built-in/custom and sent/named everywhere; and the boolean getter is customUpdater(), like skipStore() and keepBinary() next to it. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamProcessor.java | 2 +- .../datastreamer/DataStreamerReceiverMessage.java | 12 +++++------- .../processors/datastreamer/DataStreamerRequest.java | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index 668fcaa53e6f9..42fa9112a2f8b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -234,7 +234,7 @@ private void processRequest(final UUID nodeId, final DataStreamerRequest req) { updater = req.updater(); - if (req.hasCustomUpdater()) + if (req.customUpdater()) ctx.resource().injectGeneric(updater); } catch (IgniteCheckedException e) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index a264b0cf0fdb1..287a715665183 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -28,18 +28,16 @@ @UseBinaryMarshaller public class DataStreamerReceiverMessage implements Message { /** Custom cache receiver/updater; {@code null} when {@link #builtIn} is effective. */ - @Nullable @Marshalled("rcvrBytes") - StreamReceiver rcvr; + @Nullable StreamReceiver rcvr; /** Serialized {@link #rcvr}. */ @Order(0) - volatile byte[] rcvrBytes; + volatile @Nullable byte[] rcvrBytes; /** A built-in updater every node has; {@code null} when {@link #rcvr} is effective. */ - @Nullable @Order(1) - DataStreamerBuiltInUpdater builtIn; + @Nullable DataStreamerBuiltInUpdater builtIn; /** Empty constructor for serialization purposes. */ public DataStreamerReceiverMessage() { @@ -53,7 +51,7 @@ public DataStreamerReceiverMessage() { this.rcvr = rcvr; } - /** @param builtIn A built-in updater every node has, named rather than carried. */ + /** @param builtIn Built-in updater every node has, named rather than sent. */ DataStreamerReceiverMessage(DataStreamerBuiltInUpdater builtIn) { this.builtIn = builtIn; } @@ -63,7 +61,7 @@ boolean custom() { return builtIn == null; } - /** @return Receiver: the custom one carried here, or the built-in one this message names. */ + /** @return Receiver: the custom one sent here, or the built-in one this message names. */ StreamReceiver receiver() { return builtIn == null ? rcvr : builtIn.updater(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 14125cee9c675..34c45458e2e2e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -157,8 +157,8 @@ String cacheName() { return cacheName; } - /** @return {@code True} if the request carries a custom updater rather than naming a built-in one. */ - boolean hasCustomUpdater() { + /** @return {@code True} if the request sends a custom updater rather than naming a built-in one. */ + boolean customUpdater() { return updaterMsg.custom(); } From f1b7bfa2a76280585f655012adc063a80b77d1a7 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 20:46:00 +0300 Subject: [PATCH 31/35] IGNITE-27977 Review: do not keep the message in the enum A constant hands out a new message instead of caching one, so there is no shared field to publish and no race to reason about. The streamer keeps the message it builds for a receiver, so the only allocation left is one per batch while the default receiver is in use - reading the receiver itself no longer goes through a message at all, since allowOverwrite() is asked once per entry. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamerBuiltInUpdater.java | 12 +++--------- .../processors/datastreamer/DataStreamerImpl.java | 8 +++++--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java index 85d2c869afc63..56076f1e97bc0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -22,7 +22,7 @@ /** * The built-in updaters. Every node has them, so a request names the one it needs instead of carrying a serialized - * copy. The message naming one is built on demand: a node that streams with a custom receiver builds none. + * copy. */ enum DataStreamerBuiltInUpdater { /** {@link DataStreamerImpl#ISOLATED_UPDATER}. */ @@ -40,9 +40,6 @@ enum DataStreamerBuiltInUpdater { /** */ private final StreamReceiver updater; - /** */ - private DataStreamerReceiverMessage msg; - /** @param updater Updater this constant stands for. */ DataStreamerBuiltInUpdater(StreamReceiver updater) { this.updater = updater; @@ -53,12 +50,9 @@ enum DataStreamerBuiltInUpdater { return updater; } - /** @return Message naming this updater; one per constant, as a built-in updater never changes. */ + /** @return New message naming this updater. */ DataStreamerReceiverMessage message() { - if (msg == null) - msg = new DataStreamerReceiverMessage(this); - - return msg; + return new DataStreamerReceiverMessage(this); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 7c63cc3578054..9f07539f69ab7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -491,17 +491,19 @@ public IgniteInternalFuture internalFuture() { rcvrMsg = builtIn != null ? builtIn.message() : new DataStreamerReceiverMessage(rcvr); } - /** @return Message of the receiver in use, the Isolated updater until one is set. */ + /** @return Message of the receiver in use, a new one naming the Isolated updater until a receiver is set. */ private DataStreamerReceiverMessage receiverMessage() { DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; return rcvrMsg0 != null ? rcvrMsg0 : DataStreamerBuiltInUpdater.ISOLATED.message(); } - /** @return Cache receiver. */ + /** @return Cache receiver, the Isolated updater until one is set. */ @SuppressWarnings("unchecked") private StreamReceiver receiver() { - return (StreamReceiver)receiverMessage().receiver(); + DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; + + return (StreamReceiver)(rcvrMsg0 == null ? ISOLATED_UPDATER : rcvrMsg0.receiver()); } /** {@inheritDoc} */ From 2630f150473afd21cb8ee30f79a93ddaf17c0732 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 20:56:58 +0300 Subject: [PATCH 32/35] IGNITE-27977 Review: sent/name notation, and null spelled out Wording is custom/built-in and sent-as-a-name everywhere, in the carrier, the streamer and the tests. The streamer field says what null means and is marked, and the serialized bytes say when they are absent. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerImpl.java | 8 ++++---- .../datastreamer/DataStreamerReceiverMessage.java | 6 +++--- .../datastreamer/DataStreamerImplSelfTest.java | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index 9f07539f69ab7..c9115f309bc8f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -152,8 +152,8 @@ public class DataStreamerImpl implements IgniteDataStreamer, Delayed /** Amount of permissions should be available to continue new data processing. */ private static final int REMAP_SEMAPHORE_PERMISSIONS_COUNT = Integer.MAX_VALUE; - /** Cache receiver in the message that carries it; {@code null} while none is set. */ - private volatile DataStreamerReceiverMessage rcvrMsg; + /** Message of the cache receiver; {@code null} until a receiver is set, the Isolated updater being used so far. */ + private volatile @Nullable DataStreamerReceiverMessage rcvrMsg; /** IO policy resovler for data load request. */ private IgniteClosure ioPlcRslvr; @@ -491,14 +491,14 @@ public IgniteInternalFuture internalFuture() { rcvrMsg = builtIn != null ? builtIn.message() : new DataStreamerReceiverMessage(rcvr); } - /** @return Message of the receiver in use, a new one naming the Isolated updater until a receiver is set. */ + /** @return Message of the receiver in use, one of the Isolated updater until a receiver is set. */ private DataStreamerReceiverMessage receiverMessage() { DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; return rcvrMsg0 != null ? rcvrMsg0 : DataStreamerBuiltInUpdater.ISOLATED.message(); } - /** @return Cache receiver, the Isolated updater until one is set. */ + /** @return Cache receiver, custom or built-in, the Isolated updater until one is set. */ @SuppressWarnings("unchecked") private StreamReceiver receiver() { DataStreamerReceiverMessage rcvrMsg0 = rcvrMsg; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index 287a715665183..0a2bf233c2e7a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -31,7 +31,7 @@ public class DataStreamerReceiverMessage implements Message { @Marshalled("rcvrBytes") @Nullable StreamReceiver rcvr; - /** Serialized {@link #rcvr}. */ + /** Serialized {@link #rcvr}; {@code null} when {@link #builtIn} is effective. */ @Order(0) volatile @Nullable byte[] rcvrBytes; @@ -51,7 +51,7 @@ public DataStreamerReceiverMessage() { this.rcvr = rcvr; } - /** @param builtIn Built-in updater every node has, named rather than sent. */ + /** @param builtIn Built-in updater, sent as a name rather than as a copy. */ DataStreamerReceiverMessage(DataStreamerBuiltInUpdater builtIn) { this.builtIn = builtIn; } @@ -61,7 +61,7 @@ boolean custom() { return builtIn == null; } - /** @return Receiver: the custom one sent here, or the built-in one this message names. */ + /** @return Receiver: the custom one sent here, or the built-in one of this node. */ StreamReceiver receiver() { return builtIn == null ? rcvr : builtIn.updater(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index ea6f3a45d0a33..6d95c5786ddc4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -160,11 +160,11 @@ public void testReceiverMarshalledOncePerStreamer() throws Exception { assertEquals("The receiver was marshalled more than once", 1, sentUpdaters.size()); - assertTrue("The receiver was named instead of sent", F.first(sentUpdaters) instanceof byte[]); + assertTrue("A custom receiver must be sent, not named", F.first(sentUpdaters) instanceof byte[]); } /** - * Every built-in updater is named rather than sent, and the data still lands. + * Every built-in updater is sent as a name rather than as a copy, and the data still lands. * * @throws Exception If failed. */ @@ -173,7 +173,7 @@ public void testBuiltInUpdaterIsNotSent() throws Exception { for (DataStreamerBuiltInUpdater builtIn : DataStreamerBuiltInUpdater.values()) { startGridsAndStream(builtIn.updater()); - assertEquals("Expected " + builtIn + " to be named, not sent", Collections.singleton(builtIn), + assertEquals("Expected " + builtIn + " to be sent as a name", Collections.singleton(builtIn), sentUpdaters); IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); @@ -187,8 +187,8 @@ public void testBuiltInUpdaterIsNotSent() throws Exception { /** * Starts two nodes and streams {@link #KEYS_COUNT} entries from the first one, a request per entry, collecting - * the updaters they carry. Waits for the partition map first: until it is ready every partition is primary here, - * and a streamer that overwrites sends nothing to the remote node. + * the updaters they carry, custom or built-in. Waits for the partition map first: until it is ready every + * partition is primary here, and a streamer that overwrites sends nothing to the remote node. * * @param rcvr Receiver to stream with. * @throws Exception If failed. @@ -735,7 +735,7 @@ private CacheConfiguration cacheConfiguration() { return cacheCfg; } - /** A custom receiver: unlike the built-in ones, it travels with the requests. */ + /** A custom receiver: unlike the built-in ones, it is sent with the requests. */ private static class TestReceiver implements StreamReceiver { /** */ private static final long serialVersionUID = 0L; From fa1275d5f6099f0360098951c03e626518cdbd31 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 21:09:25 +0300 Subject: [PATCH 33/35] IGNITE-27977 Recognize a built-in updater by class, not by instance These updaters hold no state, so any instance of one is that updater. The lookup no longer depends on DataStreamerCacheUpdaters handing out the same object every time, which is a decision that class should be free to make. Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerBuiltInUpdater.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java index 56076f1e97bc0..a98122e8c489e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerBuiltInUpdater.java @@ -56,12 +56,14 @@ DataStreamerReceiverMessage message() { } /** + * Matches by class, so an updater built anew is still recognized as built-in: these updaters hold no state. + * * @param updater Updater to look up. * @return Constant standing for {@code updater}, or {@code null} when it is a custom one. */ static @Nullable DataStreamerBuiltInUpdater of(StreamReceiver updater) { for (DataStreamerBuiltInUpdater builtIn : values()) { - if (builtIn.updater == updater) + if (builtIn.updater.getClass() == updater.getClass()) return builtIn; } From 5de5daf481180005a2419209dc92859f9e9e0d1d Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 21:29:15 +0300 Subject: [PATCH 34/35] IGNITE-27977 Review: customUpdater() on the carrier too Co-Authored-By: Claude Opus 5 --- .../processors/datastreamer/DataStreamerReceiverMessage.java | 2 +- .../internal/processors/datastreamer/DataStreamerRequest.java | 2 +- .../processors/datastreamer/DataStreamerImplSelfTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java index 0a2bf233c2e7a..6eafdb8b5b08b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerReceiverMessage.java @@ -57,7 +57,7 @@ public DataStreamerReceiverMessage() { } /** @return {@code True} if this is a custom receiver, {@code false} if a built-in one. */ - boolean custom() { + boolean customUpdater() { return builtIn == null; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index 34c45458e2e2e..f690eac5f3864 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -159,7 +159,7 @@ String cacheName() { /** @return {@code True} if the request sends a custom updater rather than naming a built-in one. */ boolean customUpdater() { - return updaterMsg.custom(); + return updaterMsg.customUpdater(); } /** @return Updater. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 6d95c5786ddc4..d2bb0aad9ed49 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -759,7 +759,7 @@ private class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { if (sentMsg instanceof DataStreamerRequest) { DataStreamerReceiverMessage updaterMsg = ((DataStreamerRequest)sentMsg).updaterMsg; - sentUpdaters.add(updaterMsg.custom() ? updaterMsg.rcvrBytes : updaterMsg.builtIn); + sentUpdaters.add(updaterMsg.customUpdater() ? updaterMsg.rcvrBytes : updaterMsg.builtIn); } // Send stale topology only in the first request to avoid indefinitely getting failures. From 3f1ba07fc43496893d01588553b7ccd302910b52 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 10 Aug 2026 21:34:46 +0300 Subject: [PATCH 35/35] IGNITE-27977 Review: back to a static collector in the test needStaleTop is static already, so the SPI stays a static class and the collected updaters go back to a static field, named as one. Co-Authored-By: Claude Opus 5 --- .../datastreamer/DataStreamerImplSelfTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index d2bb0aad9ed49..f38ccc03b31f0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -92,7 +92,7 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { private static boolean needStaleTop = false; /** Distinct updaters sent since the current test started: the serialized bytes, or the built-in constant. */ - private final Set sentUpdaters = Collections.synchronizedSet(new HashSet<>()); + private static final Set SENT_UPDATERS = Collections.synchronizedSet(new HashSet<>()); /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { @@ -158,9 +158,9 @@ public void testCloseWithCancellation() throws Exception { public void testReceiverMarshalledOncePerStreamer() throws Exception { startGridsAndStream(new TestReceiver()); - assertEquals("The receiver was marshalled more than once", 1, sentUpdaters.size()); + assertEquals("The receiver was marshalled more than once", 1, SENT_UPDATERS.size()); - assertTrue("A custom receiver must be sent, not named", F.first(sentUpdaters) instanceof byte[]); + assertTrue("A custom receiver must be sent, not named", F.first(SENT_UPDATERS) instanceof byte[]); } /** @@ -174,7 +174,7 @@ public void testBuiltInUpdaterIsNotSent() throws Exception { startGridsAndStream(builtIn.updater()); assertEquals("Expected " + builtIn + " to be sent as a name", Collections.singleton(builtIn), - sentUpdaters); + SENT_UPDATERS); IgniteCache cache = grid(1).cache(DEFAULT_CACHE_NAME); @@ -201,7 +201,7 @@ private void startGridsAndStream(StreamReceiver rcvr) throws Exception { awaitPartitionMapExchange(); - sentUpdaters.clear(); + SENT_UPDATERS.clear(); try (IgniteDataStreamer ldr = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { ldr.receiver((StreamReceiver)rcvr); @@ -750,7 +750,7 @@ private static class TestReceiver implements StreamReceiver { /** * Simulate stale (not up-to-date) topology */ - private class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { + private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { Message sentMsg = msg instanceof GridIoMessage ? ((GridIoMessage)msg).message() : null; @@ -759,7 +759,7 @@ private class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { if (sentMsg instanceof DataStreamerRequest) { DataStreamerReceiverMessage updaterMsg = ((DataStreamerRequest)sentMsg).updaterMsg; - sentUpdaters.add(updaterMsg.customUpdater() ? updaterMsg.rcvrBytes : updaterMsg.builtIn); + SENT_UPDATERS.add(updaterMsg.customUpdater() ? updaterMsg.rcvrBytes : updaterMsg.builtIn); } // Send stale topology only in the first request to avoid indefinitely getting failures.