+ * The request bypasses the anti-thrash re-arm floor for that one swap and + * does not lower it: the floor only ever rises, so a scheduled manual + * reset cannot re-open the automatic recycle's doubling ladder. + *
+ * Also a permanent no-op on a sender configured with + * {@code symbol_dict_reset=off} ({@link LineSenderBuilder#symbolDictReset(boolean)}): + * that knob gates the arming path this request feeds, so the request is + * accepted and never acted on. + *
+ * Call on the producing thread only: like every other {@code Sender}
+ * method, this mutates producer-side state and is not thread-safe.
+ */
+ default void resetSymbolDictionary() {
+ }
+
/**
* Clear the internal buffers, discarding any unsent data.
*
@@ -1088,6 +1119,9 @@ final class LineSenderBuilder {
private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY;
private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY;
private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY;
+ private boolean symbolDictReset = QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_ENABLED;
+ private int symbolDictResetThreshold = PARAMETER_NOT_SET_EXPLICITLY;
+ private long symbolDictResetMaxWaitMillis = PARAMETER_NOT_SET_EXPLICITLY;
private String httpPath;
private String httpSettingsPath;
private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY;
@@ -1547,6 +1581,12 @@ public Sender build() {
catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY
? catchUpCapGapMinEscalationWindowMillis
: CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS;
+ int actualSymbolDictResetThreshold = symbolDictResetThreshold != PARAMETER_NOT_SET_EXPLICITLY
+ ? symbolDictResetThreshold
+ : QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS;
+ long actualSymbolDictResetMaxWaitMillis = symbolDictResetMaxWaitMillis != PARAMETER_NOT_SET_EXPLICITLY
+ ? symbolDictResetMaxWaitMillis
+ : QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS;
// sfDir is the parent (group root); the actual slot lives
// under sfDir/senderId. This is what the engine sees — the
@@ -1601,77 +1641,21 @@ public Sender build() {
try (SlotLock logicalSlotLock = slotPath == null
? null
: SlotLock.acquireLogical(slotPath)) {
- // The constructor's own recovery seed can also fail terminally, and
- // not only as UnreplayableSlotException: when SegmentRing.openExisting
- // had to skip an unreadable segment it throws SfRecoveryException (it
- // constructs UnreplayableSlotException nowhere), and where it cannot
- // even prove the chain's identity -- no manifest -- it quarantines the
- // corrupt files and returns an EMPTY recovery rather than refusing.
- // Either way the frame range cannot be shown already-acked, so recovery
- // sets the slot aside rather than risk seeding the ack cursor past
- // frames that were never delivered. All three types below are load
- // bearing; narrowing this catch to UnreplayableSlotException would
- // restore the permanent build() brick for the segment-skip case. That verdict gets
- // the exact same quarantine-and-continue treatment as the connect()-time
- // verdict below -- constructing cursorEngine is not inside the loop below,
- // so a throw here would otherwise escape build() entirely, uncaught.
- // quarantineTornSlot(null, ...) renames the WHOLE slot directory aside
- // (not just the unreadable segment file) before building the replacement
- // at the original slotPath, so the replacement starts on a genuinely empty
- // directory with nothing left to skip -- it cannot throw the same way
- // twice, which is what makes looping unnecessary here.
- boolean quarantined = false;
- CursorSendEngine cursorEngine;
- try {
- try {
- cursorEngine = new CursorSendEngine(
- slotPath, actualSfMaxSegmentBytes,
- actualSfMaxTotalBytes, actualSfAppendDeadlineNanos,
- actualSfSyncIntervalNanos);
- } catch (SfSanitizedResidueException first) {
- // NOT terminal, and it must be intercepted ahead of its
- // SfRecoveryException parent below. Recovery durably zeroed
- // proven-dead sealed residue BEFORE failing closed, so the
- // chain on disk is already healed: quarantining here would
- // set aside a slot whose backlog replays perfectly. Retry
- // once over the healed chain; a repeat is genuine and takes
- // the terminal arm.
- LOG.info("sf slot {}: sealed residue sanitized during recovery ({}); "
- + "retrying over the healed chain",
- slotPath, first.getMessage());
- cursorEngine = new CursorSendEngine(
- slotPath, actualSfMaxSegmentBytes,
- actualSfMaxTotalBytes, actualSfAppendDeadlineNanos,
- actualSfSyncIntervalNanos);
- }
- } catch (UnreplayableSlotException | SfRecoveryException
- | MmapSegmentCorruptionException e) {
- // The terminal recovery verdicts, and the only ones build()
- // sets a slot aside for. UnreplayableSlotException says the
- // symbol dictionary cannot be rebuilt from any source;
- // SfRecoveryException and MmapSegmentCorruptionException say
- // the durable chain itself is proven corrupt or incomplete.
- // None of the three clears on a retry, and senderId is stable
- // with a not-fully-drained slot retained on close -- so
- // without this arm every restart re-recovers the same slot and
- // throws again, and the application cannot construct a Sender
- // at all, not even to BUFFER new rows.
- //
- // Deliberately NOT catching plain MmapSegmentException or
- // SfOperationalException: those are operational (EMFILE,
- // ENOMEM, an unreadable-but-possibly-intact file). Aborting
- // startup on them is correct; quarantining on them would
- // convert a transient into the permanent loss of a healthy
- // slot's durable frames.
- if (slotPath == null) {
- throw e;
- }
- quarantined = true;
- cursorEngine = quarantineTornSlot(
- null, e, sfDir, senderId, slotPath, actualSfMaxSegmentBytes,
- actualSfMaxTotalBytes, actualSfAppendDeadlineNanos,
- actualSfSyncIntervalNanos, errorHandler);
- }
+ // Recovery-verdict handling lives in constructEngineOnSlotLocked.
+ ConstructedEngine constructed = constructEngineOnSlotLocked(
+ sfDir, senderId, slotPath,
+ actualSfMaxSegmentBytes, actualSfMaxTotalBytes,
+ actualSfAppendDeadlineNanos, actualSfSyncIntervalNanos,
+ errorHandler);
+ // Seeded from constructEngineOnSlotLocked's own verdict, not
+ // hardcoded false: if construction already quarantined this
+ // slot, the connect loop below must count that as the one
+ // quarantine build() allows per attempt (see its "quarantined
+ // || slotPath == null" guard) rather than starting blind and
+ // risking a second quarantineTornSlot pass on what should be
+ // an immediate close-and-rethrow.
+ boolean quarantined = constructed.quarantined;
+ CursorSendEngine cursorEngine = constructed.engine;
int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY
? errorInboxCapacity
: io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY;
@@ -1714,7 +1698,10 @@ public Sender build() {
actualConnectionListenerInboxCapacity,
actualMaxFrameRejections,
actualPoisonMinEscalationWindowMillis,
- actualCatchUpCapGapMinEscalationWindowMillis
+ actualCatchUpCapGapMinEscalationWindowMillis,
+ symbolDictReset,
+ actualSymbolDictResetThreshold,
+ actualSymbolDictResetMaxWaitMillis
);
} catch (UnreplayableSlotException e) {
// The one failure build() recovers from. The slot's frames reference ids
@@ -1767,6 +1754,24 @@ public Sender build() {
// dispatcher daemon, drainer pool, microbatch buffers and
// WebSocketClient inside the abandoned `connected`.
connected.setTransactional(transactional);
+ final String rebuildSfDir = sfDir;
+ final String rebuildSenderId = senderId;
+ final SenderErrorHandler buildTimeHandler = errorHandler;
+ connected.setEngineRebuildFactory(new QwpWebSocketSender.EngineRebuildFactory() {
+ @Override
+ public CursorSendEngine rebuild() {
+ return rebuild(buildTimeHandler);
+ }
+
+ @Override
+ public CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
+ return LineSenderBuilder.constructEngineOnSlot(
+ rebuildSfDir, rebuildSenderId, slotPath,
+ actualSfMaxSegmentBytes, actualSfMaxTotalBytes,
+ actualSfAppendDeadlineNanos, actualSfSyncIntervalNanos,
+ liveHandler);
+ }
+ });
try {
// Install the drainer listener BEFORE startOrphanDrainers
// below: drainers must see the listener at submit time so
@@ -1902,6 +1907,116 @@ public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) {
return this;
}
+ /**
+ * Enables periodic recycling (rebuilding) of the sender's symbol dictionary
+ * once it reaches {@link #symbolDictResetThreshold(int)} distinct symbols,
+ * so a long-lived sender's dictionary does not grow without bound.
+ *
+ * The recycle itself runs at a {@code table()} call that finds the backlog + * already acknowledged. A recycle armed longer than + * {@link #symbolDictResetMaxWaitMillis(long)} without such a call pauses + * one {@code table()} call for up to that long to drain the backlog; if + * the backlog still does not drain, the recycle stays armed and under + * sustained saturation may be deferred indefinitely (see + * {@link Sender#resetSymbolDictionary()}). + *
+ * Switching it off also disables the manual valve: + * {@link Sender#resetSymbolDictionary()} becomes a permanent no-op, + * because arming gates on this knob. + *
+ * Switching it off removes the only bound on dictionary growth below + * the hard cap ({@link io.questdb.client.cutlass.qwp.protocol.QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE}, + * 2,000,000). Servers released before QuestDB 10.0.0 cap the + * dictionary at 1,000,000 and reject anything beyond it as a terminal + * parse error, so with the recycle off against a pre-10.0.0 server + * keep symbol cardinality below 1M. + *
+ * Default {@code true} (on). WebSocket transport only. + */ + public LineSenderBuilder symbolDictReset(boolean enabled) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("symbol_dict_reset is only supported for WebSocket transport"); + } + this.symbolDictReset = enabled; + return this; + } + + /** + * Number of distinct symbols the sender's dictionary may accumulate before + * {@link #symbolDictReset(boolean)} triggers a recycle. Each recycle raises + * the effective bar to {@code max(threshold, 2 x dictionary size at the swap)}, + * capped at half of {@link QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE}, so a + * bounded live set larger than the threshold recycles once and settles + * instead of recycling on every refill. The bar never drops, a manual + * {@link Sender#resetSymbolDictionary()} swap included. Must be greater than + * {@code 0} and no larger than half of {@link QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE} + * (the re-arm floor's own cap): arming happens at a flush tail and the swap at the + * next drained {@code table(...)} call, so a threshold nearer the protocol cap would + * hit the cap error before any recycle could run. + *
+ * Default {@code 100_000}. WebSocket transport only. + */ + public LineSenderBuilder symbolDictResetThreshold(int threshold) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("symbol_dict_reset_threshold is only supported for WebSocket transport"); + } + if (threshold <= 0 || threshold > QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2) { + throw new LineSenderException("symbol_dict_reset_threshold must be > 0 and <= ") + .put(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2).put(": ").put(threshold); + } + this.symbolDictResetThreshold = threshold; + return this; + } + + /** + * Upper bound, in milliseconds, on how long a triggered symbol-dictionary + * recycle stays armed before it may block the calling thread to force + * progress. Once a recycle has been armed for longer than this window + * without an opportunistic (idle) drain, the NEXT row-start call + * ({@code table(...)}) BLOCKS the producing thread for up to this many + * millis waiting for the outstanding backlog to drain, then recycles + * before returning. A producer that keeps frames in flight at every row + * start never exposes a drained instant on its own; on a healthy link + * the pause is about one acknowledgement round trip, because the paused + * producer stops refilling the backlog. If the backlog still has not + * drained by the deadline (an outage, or a producer that outruns the + * wire), that call gives up (logging a warning) and the recycle stays + * armed for a later opportunistic retry. At most one blocking wait + * happens per armed window, so an outage costs the producer one bounded + * pause, and a timeout during that one wait leaves the recycle waiting + * for a row start that finds the backlog drained on its own. + *
+ * {@code 0} disables blocking entirely (opportunistic-only): the recycle + * then only ever runs when a {@code table(...)} call finds the backlog + * already drained, and under sustained load it may be deferred + * indefinitely. To detect a recycle that never finds its drained + * instant, sample {@code QwpWebSocketSender.isResetArmed()} together + * with {@code getSymbolDictEpoch()} and + * {@code getSymbolDictResetStarvationTimeouts()}: armed staying + * {@code true} while the epoch does not advance means no row start + * observes a drained backlog. Either raise this value, or drain + * explicitly ({@code drain(...)}) at a quiet point of your choosing. + *
+ * The default is kept below the sender pool's acquire timeout: a pooled + * sender inherits an armed recycle at give-back, and the next borrower + * may pay this wait while holding its lease. + *
+ * Default {@code 2_000}. WebSocket transport only.
+ */
+ public LineSenderBuilder symbolDictResetMaxWaitMillis(long maxWaitMillis) {
+ if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport");
+ }
+ if (maxWaitMillis < 0) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis must be >= 0: ").put(maxWaitMillis);
+ }
+ if (maxWaitMillis > Long.MAX_VALUE / 1_000_000L) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis is out of range: ").put(maxWaitMillis);
+ }
+ this.symbolDictResetMaxWaitMillis = maxWaitMillis;
+ return this;
+ }
+
/**
* close() drain timeout in milliseconds. The sender's {@code close()}
* method blocks up to this many millis waiting for the server to ACK
@@ -3199,6 +3314,135 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na
}
}
+ /**
+ * Result of {@link #constructEngineOnSlotLocked}: the constructed engine, plus
+ * whether construction itself had to quarantine a torn slot to produce it.
+ * {@link #build} folds {@code quarantined} into its own connect-loop retry
+ * guard, so a construction-time quarantine still counts toward the one
+ * quarantine build() allows per attempt -- the invariant a single shared
+ * {@code quarantined} local enforced before this method existed.
+ */
+ static final class ConstructedEngine {
+ final CursorSendEngine engine;
+ final boolean quarantined;
+
+ ConstructedEngine(CursorSendEngine engine, boolean quarantined) {
+ this.engine = engine;
+ this.quarantined = quarantined;
+ }
+ }
+
+ /**
+ * Constructs a {@code CursorSendEngine} on {@code slotPath}, quarantining a torn
+ * slot exactly as {@link #build}'s connect loop does when the constructor itself
+ * hits a terminal recovery verdict. Assumes the caller already holds
+ * {@code slotPath}'s logical lock (or {@code slotPath == null}, memory mode).
+ */
+ static ConstructedEngine constructEngineOnSlotLocked(
+ String sfDir, String senderId, String slotPath,
+ long maxSegmentBytes, long maxTotalBytes,
+ long appendDeadlineNanos, long syncIntervalNanos,
+ SenderErrorHandler errorHandler) {
+ // The constructor's own recovery seed can also fail terminally, and
+ // not only as UnreplayableSlotException: when SegmentRing.openExisting
+ // had to skip an unreadable segment it throws SfRecoveryException (it
+ // constructs UnreplayableSlotException nowhere), and where it cannot
+ // even prove the chain's identity -- no manifest -- it quarantines the
+ // corrupt files and returns an EMPTY recovery rather than refusing.
+ // Either way the frame range cannot be shown already-acked, so recovery
+ // sets the slot aside rather than risk seeding the ack cursor past
+ // frames that were never delivered. All three types below are load
+ // bearing; narrowing this catch to UnreplayableSlotException would
+ // restore the permanent build() brick for the segment-skip case. That verdict gets
+ // the same quarantine-and-continue treatment as build()'s connect()-time
+ // verdict; for build() the construction runs outside its retry loop, and
+ // for a recycle rebuild there is no loop at all.
+ // quarantineTornSlot(null, ...) renames the WHOLE slot directory aside
+ // (not just the unreadable segment file) before building the replacement
+ // at the original slotPath, so the replacement starts on a genuinely empty
+ // directory with nothing left to skip -- it cannot throw the same way
+ // twice, which is what makes looping unnecessary here.
+ boolean quarantined = false;
+ CursorSendEngine cursorEngine;
+ try {
+ try {
+ cursorEngine = new CursorSendEngine(
+ slotPath, maxSegmentBytes,
+ maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos);
+ } catch (SfSanitizedResidueException first) {
+ // NOT terminal, and it must be intercepted ahead of its
+ // SfRecoveryException parent below. Recovery durably zeroed
+ // proven-dead sealed residue BEFORE failing closed, so the
+ // chain on disk is already healed: quarantining here would
+ // set aside a slot whose backlog replays perfectly. Retry
+ // once over the healed chain; a repeat is genuine and takes
+ // the terminal arm.
+ LOG.info("sf slot {}: sealed residue sanitized during recovery ({}); "
+ + "retrying over the healed chain",
+ slotPath, first.getMessage());
+ cursorEngine = new CursorSendEngine(
+ slotPath, maxSegmentBytes,
+ maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos);
+ }
+ } catch (UnreplayableSlotException | SfRecoveryException
+ | MmapSegmentCorruptionException e) {
+ // The terminal recovery verdicts, and the only ones build()
+ // sets a slot aside for. UnreplayableSlotException says the
+ // symbol dictionary cannot be rebuilt from any source;
+ // SfRecoveryException and MmapSegmentCorruptionException say
+ // the durable chain itself is proven corrupt or incomplete.
+ // None of the three clears on a retry, and senderId is stable
+ // with a not-fully-drained slot retained on close -- so
+ // without this arm every restart re-recovers the same slot and
+ // throws again, and the application cannot construct a Sender
+ // at all, not even to BUFFER new rows.
+ //
+ // Deliberately NOT catching plain MmapSegmentException or
+ // SfOperationalException: those are operational (EMFILE,
+ // ENOMEM, an unreadable-but-possibly-intact file). Aborting
+ // startup on them is correct; quarantining on them would
+ // convert a transient into the permanent loss of a healthy
+ // slot's durable frames.
+ if (slotPath == null) {
+ throw e;
+ }
+ quarantined = true;
+ cursorEngine = quarantineTornSlot(
+ null, e, sfDir, senderId, slotPath, maxSegmentBytes,
+ maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos, errorHandler);
+ }
+ return new ConstructedEngine(cursorEngine, quarantined);
+ }
+
+ /**
+ * {@link #constructEngineOnSlotLocked} wrapped in its own narrow acquisition of
+ * {@code slotPath}'s logical lock. {@link #build} itself does not call this --
+ * its own lock spans the connect loop too, see the comment at its call site --
+ * this entry point is for callers that only need a freshly (re)built engine on
+ * an already-owned slot, such as a symbol-dictionary epoch rebuild. Recovery
+ * verdicts still quarantine here exactly as they do under {@link #build} --
+ * that happens inside {@link #constructEngineOnSlotLocked}. Only the
+ * quarantined FLAG is discarded: it exists to seed {@code build}'s connect-loop
+ * retry guard, and a recycle rebuild has no such loop. A rebuild that fails
+ * outright does not latch the sender terminal either; the recycle abandons and
+ * retries on the next send.
+ */
+ static CursorSendEngine constructEngineOnSlot(
+ String sfDir, String senderId, String slotPath,
+ long maxSegmentBytes, long maxTotalBytes,
+ long appendDeadlineNanos, long syncIntervalNanos,
+ SenderErrorHandler errorHandler) {
+ try (SlotLock logicalSlotLock = slotPath == null
+ ? null : SlotLock.acquireLogical(slotPath)) {
+ return constructEngineOnSlotLocked(sfDir, senderId, slotPath,
+ maxSegmentBytes, maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos, errorHandler).engine;
+ }
+ }
+
/**
* Sets a slot aside that either connect() (a symbol dictionary that cannot cover its
* surviving frames, {@code UnreplayableSlotException}) or the
@@ -3307,10 +3551,11 @@ private static CursorSendEngine quarantineTornSlot(
// caller must be able to act on, and LOG.error alone cannot carry it: this client
// ships slf4j-api with no binding, so an embedding app with no provider gets a NOP
// logger and the loss is announced nowhere. Deliver it programmatically too, so an
- // errorHandler can alert / page / record it. Dispatched synchronously here because
- // the async SenderErrorDispatcher belongs to the connected sender, which does not
- // exist yet at build time. A throwing handler must not turn a contained outage back
- // into a failed build, so swallow anything it raises.
+ // errorHandler can alert / page / record it. Dispatched synchronously here: at
+ // build time the async SenderErrorDispatcher does not exist yet, and at a recycle
+ // rebuild a data-loss notice must not be dropped under its inbox pressure. A
+ // throwing handler must not turn a contained outage back into a failed build, so
+ // swallow anything it raises.
if (errorHandler != null) {
try {
errorHandler.onError(SenderError.dataLoss(
@@ -3866,6 +4111,30 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
}
pos = getValue(configurationString, pos, sink, "catch_up_cap_gap_min_escalation_window_millis");
catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catch_up_cap_gap_min_escalation_window_millis"));
+ } else if (Chars.equals("symbol_dict_reset", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset");
+ if (Chars.equalsIgnoreCase("on", sink)) {
+ symbolDictReset(true);
+ } else if (Chars.equalsIgnoreCase("off", sink)) {
+ symbolDictReset(false);
+ } else {
+ throw new LineSenderException("invalid symbol_dict_reset [value=").put(sink).put(", allowed-values=[on, off]]");
+ }
+ } else if (Chars.equals("symbol_dict_reset_threshold", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_threshold is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset_threshold");
+ symbolDictResetThreshold(parseIntValue(sink, "symbol_dict_reset_threshold"));
+ } else if (Chars.equals("symbol_dict_reset_max_wait_millis", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset_max_wait_millis");
+ symbolDictResetMaxWaitMillis(parseLongValue(sink, "symbol_dict_reset_max_wait_millis"));
} else if (Chars.equals("initial_connect_retry", sink)) {
if (protocol != PROTOCOL_WEBSOCKET) {
throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport");
@@ -4141,6 +4410,12 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString)
if (view.has("catch_up_cap_gap_min_escalation_window_millis")) {
catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catch_up_cap_gap_min_escalation_window_millis"));
}
+ if (view.has("symbol_dict_reset_threshold")) {
+ symbolDictResetThreshold(wsInt(view, v, "symbol_dict_reset_threshold"));
+ }
+ if (view.has("symbol_dict_reset_max_wait_millis")) {
+ symbolDictResetMaxWaitMillis(wsLong(view, v, "symbol_dict_reset_max_wait_millis"));
+ }
if (view.has("sf_append_deadline_millis")) {
sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis"));
}
@@ -4210,6 +4485,16 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString)
throw new LineSenderException("invalid initial_connect_retry [value=").put(s).put(", allowed-values=[on, off, true, false, sync, async]]");
}
}
+ s = view.getStr("symbol_dict_reset");
+ if (s != null) {
+ if (s.equalsIgnoreCase("on")) {
+ symbolDictReset(true);
+ } else if (s.equalsIgnoreCase("off")) {
+ symbolDictReset(false);
+ } else {
+ throw new LineSenderException("invalid symbol_dict_reset [value=").put(s).put(", allowed-values=[on, off]]");
+ }
+ }
return this;
} catch (IllegalArgumentException e) {
throw new LineSenderException(e.getMessage());
@@ -4325,6 +4610,9 @@ public java.util.Map
* A rotating credential goes to {@code connectWithCredentialSupplier} instead, which carries a distinct
* name precisely so this form keeps its exact descriptor and a bare {@code null} credential stays
- * unambiguous.
+ * unambiguous. Symbol-dictionary recycling defaults to
+ * {@link #DEFAULT_SYMBOL_DICT_RESET_ENABLED} / {@link #DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS} /
+ * {@link #DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS}; the overload below takes those knobs explicitly.
*/
public static QwpWebSocketSender connect(
List
@@ -889,7 +1160,10 @@ public static QwpWebSocketSender connectWithCredentialSupplier(
int connectionListenerInboxCapacity,
int maxFrameRejections,
long poisonMinEscalationWindowMillis,
- long catchUpCapGapMinEscalationWindowMillis
+ long catchUpCapGapMinEscalationWindowMillis,
+ boolean symbolDictResetEnabled,
+ int symbolDictResetThresholdSymbols,
+ long symbolDictResetMaxWaitMillis
) {
QwpWebSocketSender sender = new QwpWebSocketSender(
endpoints, tlsConfig,
@@ -908,6 +1182,9 @@ public static QwpWebSocketSender connectWithCredentialSupplier(
sender.maxFrameRejections = maxFrameRejections;
sender.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis;
sender.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis;
+ sender.resetEnabled = symbolDictResetEnabled;
+ sender.resetThresholdSymbols = symbolDictResetThresholdSymbols;
+ sender.resetMaxWaitMillis = symbolDictResetMaxWaitMillis;
sender.initialConnectMode = initialConnectMode == null
? Sender.InitialConnectMode.OFF
: initialConnectMode;
@@ -1069,30 +1346,50 @@ public void atNow() {
@Override
public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) {
checkNotClosed();
- if (cursorEngine == null) {
- return targetFsn < 0L;
+ checkRecycleFailure();
+ // Snapshot: the recycle transitions cursorEngine non-null -> null ->
+ // non-null on the producer thread; reading the field once keeps this
+ // method from dereferencing a half-swapped null. While it is null,
+ // anything at or below the watermark the recycle barrier proved
+ // durable is truthfully "acked".
+ CursorSendEngine engine = cursorEngine;
+ if (engine == null) {
+ return targetFsn < 0L || targetFsn <= lastRecycleDurableFsn;
}
- cursorEngine.checkDurability();
+ engine.checkDurability();
// Surface latched errors before any early-return path, so a caller
// polling with timeoutMillis <= 0 to drive their own loop sees the
// throw instead of an indefinite "not yet". The durability latch
// above is transient: it throws while latched, and clears once a
// later periodic sync pass fully succeeds so producers can resume.
- if (cursorSendLoop != null) {
- cursorSendLoop.checkError();
+ // Snapshot for the same reason as engine above: the recycle nulls
+ // cursorSendLoop on the producer thread, so a double read here could
+ // NPE between the check and the call.
+ CursorWebSocketSendLoop loop = cursorSendLoop;
+ if (loop != null) {
+ loop.checkError();
}
checkConnectionError();
- if (cursorEngine.ackedFsn() >= targetFsn) {
+ if (targetFsn >= 0) {
+ long internalTarget = targetFsn - fsnEpochBase;
+ if (internalTarget < 0) {
+ // target belongs to a pre-recycle epoch: proven acked before the swap
+ return true;
+ }
+ targetFsn = internalTarget;
+ }
+ if (engine.ackedFsn() >= targetFsn) {
return true;
}
if (timeoutMillis <= 0L) {
return false;
}
long deadlineNanos = System.nanoTime() + timeoutMillis * 1_000_000L;
- while (cursorEngine.ackedFsn() < targetFsn) {
- cursorEngine.checkDurability();
- if (cursorSendLoop != null) {
- cursorSendLoop.checkError();
+ while (engine.ackedFsn() < targetFsn) {
+ engine.checkDurability();
+ loop = cursorSendLoop;
+ if (loop != null) {
+ loop.checkError();
}
checkConnectionError();
if (System.nanoTime() >= deadlineNanos) {
@@ -1340,10 +1637,14 @@ private void close0(boolean[] restoreInterrupt) {
? cursorSendLoop.getSynchronouslySurfacedError() : null;
try {
- // Only drain when both the engine and the I/O loop are wired
- // up — close() is also called from createForTesting() teardown
- // and from connect() rollback paths where one or both may be null.
- if (connectionError.get() == null && cursorEngine != null && cursorSendLoop != null) {
+ // The flush/commit/seal trio needs only the engine: rows are
+ // encoded into the SF ring on the user thread. The loop-only
+ // members below (checkUnsurfacedError, drainOnClose) keep
+ // their own gate -- with no I/O loop nothing can advance
+ // acks, so draining would only stall for the full timeout.
+ // Also covers createForTesting() teardown and connect()
+ // rollback paths where the loop (or both) may be null.
+ if (connectionError.get() == null && cursorEngine != null) {
// 1) Flush user-thread state into the engine (encoded
// rows -> mmap'd / malloc'd ring). After this, the
// cursor engine's publishedFsn reflects the final
@@ -1409,7 +1710,7 @@ private void close0(boolean[] restoreInterrupt) {
// both still get the loud rethrow on shutdown.
boolean terminalOwnedByCustomHandler = errorDispatcher != null
&& errorDispatcher.hasDeliveredTerminalToCustomHandler();
- if (!terminalOwnedByCustomHandler) {
+ if (cursorSendLoop != null && !terminalOwnedByCustomHandler) {
cursorSendLoop.checkUnsurfacedError();
}
// 3) Bounded drain: block until the server has ACK'd
@@ -1422,7 +1723,7 @@ private void close0(boolean[] restoreInterrupt) {
// without re-throwing (re-throwing would double-signal
// an error the user already handled). Otherwise the
// drain keeps the loud safety net and surfaces it.
- if (closeFlushTimeoutMillis > 0L) {
+ if (cursorSendLoop != null && closeFlushTimeoutMillis > 0L) {
drainOnClose(terminalOwnedByCustomHandler, restoreInterrupt);
}
}
@@ -1520,8 +1821,9 @@ public boolean isCloseCleanupComplete() {
* Not a one-shot snapshot: when close() left engine cleanup pending on a
* manager-worker quiescence or I/O-thread exit path, this re-probes the
* retained engine and latches true the moment that cleanup completes — pools re-probe retired
- * slots through this getter to recover their capacity. Monotonic:
- * false→true only, never back. Cheap (volatile reads on every common
+ * slots through this getter to recover their capacity. Reset to false by a
+ * recycle that rebuilds the engine (the fresh engine holds the flock
+ * again); otherwise latches false→true. Cheap (volatile reads on every common
* path) so pools may call it under their capacity lock; only the rare
* orphaned-retry state below does more.
*
@@ -1768,6 +2070,7 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) {
*/
@Override
public void flush() {
+ checkRecycleFailure();
flushAndGetSequence();
}
@@ -1786,6 +2089,7 @@ public void flush() {
@Override
public long flushAndGetSequence() {
checkNotClosed();
+ checkRecycleFailure();
if (cursorEngine != null) {
cursorEngine.checkDurability();
}
@@ -1817,7 +2121,7 @@ public long flushAndGetSequence() {
checkConnectionError();
long afterFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L;
- return afterFsn > beforeFsn ? afterFsn : -1L;
+ return afterFsn > beforeFsn ? fsnEpochBase + afterFsn : -1L;
}
/**
@@ -1846,8 +2150,10 @@ public long flushAndGetSequence() {
*/
@Override
public boolean drain(long timeoutMillis) {
+ checkRecycleFailure();
flush();
- long targetFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L;
+ long targetRaw = cursorEngine != null ? cursorEngine.publishedFsn() : -1L;
+ long targetFsn = targetRaw < 0 ? targetRaw : fsnEpochBase + targetRaw;
return awaitAckedFsn(targetFsn, timeoutMillis);
}
@@ -1926,15 +2232,34 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va
/**
* Highest FSN that has been server-acknowledged. Rejections never advance
- * the watermark. {@code -1} if
- * the I/O loop has not yet started or no batch has been published.
+ * the watermark. Returns {@code -1} only while nothing has ever been
+ * published in this sender's lifetime. On a live sender the value never
+ * collapses back to {@code -1}: after a symbol-dictionary recycle the
+ * accessor keeps reporting the last pre-swap durable watermark until the
+ * fresh epoch publishes. (After {@code close()} the reading is
+ * unspecified.)
*
* Snapshot accessor — for a bounded wait, use
* {@link #awaitAckedFsn(long, long)}.
*/
@Override
public long getAckedFsn() {
- return cursorEngine != null ? cursorEngine.ackedFsn() : -1L;
+ // Read fsnEpochBase FIRST, then cursorEngine: the recycle writes
+ // engine=null -> base+=L+1 -> engine=fresh, so a reader that saw the
+ // NEW base is ordered after the null write and can only observe null
+ // or the fresh engine -- never (new base, stale engine), which would
+ // fabricate an FSN above anything ever published. The clamp against
+ // lastRecycleDurableFsn keeps the other torn pair (old base, fresh
+ // engine) from reading below a value already returned. Sender is
+ // documented single-threaded; this keeps best-effort monitor reads
+ // truthful rather than promising thread safety.
+ long base = fsnEpochBase;
+ Runnable witness = ackedFsnReadWitness;
+ if (witness != null) {
+ witness.run();
+ }
+ CursorSendEngine engine = cursorEngine;
+ return engine != null ? Math.max(lastRecycleDurableFsn, base + engine.ackedFsn()) : lastRecycleDurableFsn;
}
/**
@@ -1991,6 +2316,44 @@ public CursorSendEngine getCursorEngineForTesting() {
return cursorEngine;
}
+ @TestOnly
+ public int getSentMaxSymbolIdForTesting() {
+ return sentMaxSymbolId;
+ }
+
+ @TestOnly
+ public boolean hasDeferredMessagesForTesting() {
+ return hasDeferredMessages;
+ }
+
+ /**
+ * Fabricates the state a step-2 close failure leaves behind -- an
+ * abandoned recycle parked at CLOSE_LOOP -- on a loop that is genuinely
+ * closed, so the resume's re-close converges instantly instead of
+ * depending on the interrupt race the Assume-gated abandon test drives.
+ * The loop reference is deliberately KEPT (the resume re-closes it).
+ */
+ @TestOnly
+ public void forceCloseLoopAbandonForTesting() {
+ if (cursorSendLoop == null) {
+ throw new IllegalStateException("connect and publish first: the CLOSE_LOOP arm needs a loop");
+ }
+ cursorSendLoop.close();
+ hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
+ connected = false;
+ recycleResume = RecycleResume.CLOSE_LOOP;
+ }
+
+ @TestOnly
+ public void setResumeCommitFaultForTesting(Runnable fault) {
+ this.resumeCommitFaultForTesting = fault;
+ }
+
+ @TestOnly
+ public void setChunkPublishFaultForTesting(Runnable fault) {
+ this.chunkPublishFaultForTesting = fault;
+ }
+
/**
* Background orphan-drainer pool, or {@code null} when
* {@code drain_orphans} is off or no orphan slot was adopted.
@@ -2037,6 +2400,16 @@ public int getEffectiveAutoFlushBytes() {
return effectiveAutoFlushBytes;
}
+ /**
+ * The installed engine-rebuild factory, so a test can wrap the real one
+ * (e.g. fault-inject the first rebuild and delegate afterwards) instead of
+ * replacing it outright. {@code null} for a {@code connect()}-built sender.
+ */
+ @TestOnly
+ public EngineRebuildFactory getEngineRebuildFactoryForTesting() {
+ return engineRebuildFactory;
+ }
+
/**
* Snapshot of the typed payload for the latched terminal server-rejection error,
* or {@code null} if the I/O loop has not latched a server-rejection terminal
@@ -2125,6 +2498,27 @@ public int getServerMaxBatchSize() {
return serverMaxBatchSize;
}
+ /** Resolved value of {@code symbol_dict_reset_max_wait_millis}. */
+ @TestOnly
+ public long getSymbolDictResetMaxWaitMillis() {
+ return resetMaxWaitMillis;
+ }
+
+ /** Resolved value of {@code symbol_dict_reset_threshold}. */
+ @TestOnly
+ public int getSymbolDictResetThreshold() {
+ return resetThresholdSymbols;
+ }
+
+ /**
+ * The current re-arm floor: 0 before the first swap, then
+ * {@code min(2 x dictSizeAtSwap, MAX_SYMBOL_DICTIONARY_SIZE / 2)}.
+ */
+ @TestOnly
+ public int getResetFloorSymbolsForTesting() {
+ return resetFloorSymbols;
+ }
+
@TestOnly
public QwpTableBuffer getTableBuffer(String tableName) {
QwpTableBuffer buffer = tableBuffers.get(tableName);
@@ -2157,9 +2551,11 @@ public boolean isCredentialDynamic() {
/**
* Whether this sender is still in delta-encoded mode. Flips to {@code false}
- * permanently once {@link #disableDeltaDict} fires (a persisted-dictionary
- * write failure, including a recognised mmap access fault) -- every later
- * flush then ships full self-sufficient frames instead.
+ * for the rest of this epoch once {@link #disableDeltaDict} fires (a
+ * persisted-dictionary write failure, including a recognised mmap access
+ * fault) -- every later flush this epoch then ships full self-sufficient
+ * frames instead. A symbol-dictionary recycle re-derives this from the
+ * fresh engine.
*/
@TestOnly
public boolean isDeltaDictEnabledForTest() {
@@ -2167,11 +2563,115 @@ public boolean isDeltaDictEnabledForTest() {
}
/**
- * Total binary frames whose ACKs have been received and applied.
+ * Whether the symbol-dictionary recycle is currently armed. Set by
+ * {@link #armIfEligible()} at the tail of every flush (and immediately by
+ * {@link #resetSymbolDictionary()} when no row or flush is in progress).
+ */
+ public boolean isResetArmed() {
+ return resetArmed;
+ }
+
+ /** Resolved value of {@code symbol_dict_reset}. */
+ @TestOnly
+ public boolean isSymbolDictResetEnabled() {
+ return resetEnabled;
+ }
+
+ /** Current value of {@link #fsnEpochBase}. */
+ @TestOnly
+ public long getFsnEpochBaseForTesting() {
+ return fsnEpochBase;
+ }
+
+ /**
+ * Number of symbol-dictionary recycles this sender has completed. Advances
+ * by one at step 6 of {@link #recycleForDictReset()}, the instant the swap
+ * commits to the new epoch -- after the engine rebuild (step 4) has
+ * already succeeded, so a step-4 rebuild failure -- which abandons the
+ * recycle to be resumed by a later send -- leaves this counter
+ * un-bumped, while a later step-7 reconnect failure (which cannot
+ * latch: the swap already committed by then) still leaves this
+ * incremented. Unlike the per-send-loop
+ * {@code getTotal*} counters, it is scoped to the sender's whole lifetime
+ * and never resets. volatile: a concurrent read sees the latest write the
+ * producer thread completed; there is no atomicity between this counter
+ * and {@link #getSymbolDictResetStarvationTimeouts()}.
+ */
+ public long getSymbolDictEpoch() {
+ return symbolDictEpoch;
+ }
+
+ /**
+ * Number of times {@link #maybeBlockForStarvedReset()} has timed out
+ * without the backlog draining. 0 until the first such timeout. volatile,
+ * written only from the producer thread inside
+ * {@link #maybeBlockForStarvedReset()}: same thread-safety caveat as
+ * {@link #getSymbolDictEpoch()} -- a concurrent read sees the latest
+ * completed write, with no atomicity across the two counters.
+ *
+ * The wait runs at most once per armed window, so this advances by at
+ * most one per recycle that had to be forced: a count that keeps growing
+ * while {@link #getSymbolDictEpoch()} keeps advancing means recycles are
+ * completing, but each one first had to pause the producer; a count that
+ * stops growing while {@link #isResetArmed()} stays {@code true} and the
+ * epoch does not advance means the one wait this armed window gets has
+ * already timed out and the recycle now depends on a row start finding
+ * the backlog drained on its own. At {@code symbol_dict_reset_max_wait_millis=0}
+ * the wait never runs and this counter is structurally 0, so 0 does NOT
+ * mean "no starvation" -- sample {@link #isResetArmed()} alongside
+ * {@link #getSymbolDictEpoch()} instead.
+ */
+ public long getSymbolDictResetStarvationTimeouts() {
+ return symbolDictResetStarvationTimeouts;
+ }
+
+ /**
+ * Test-only entry point for {@link #rollFsnEpochBase}, the same private
+ * roll the symbol-dict recycle swap calls in production once the engine
+ * rebuild has committed. See that method's precondition: {@code cursorSendLoop}
+ * must be {@code null} -- roll before the sender's first connect (e.g. via
+ * {@link #createForTesting}), never on an already-connected sender.
+ */
+ @TestOnly
+ public void rollFsnEpochBaseForTesting(long lastPublishedFsn) {
+ rollFsnEpochBase(lastPublishedFsn);
+ }
+
+ /**
+ * Advances {@link #fsnEpochBase} past every FSN handed out under the
+ * epoch that just ended. {@code lastPublishedFsn} is the highest raw FSN
+ * the outgoing cursor engine ever published ({@code -1} if it published
+ * nothing), so the next raw FSN the fresh engine hands out --
+ * {@code 0} -- maps to external {@code lastPublishedFsn + 1 + 0}, one
+ * past the last external FSN this sender ever reported.
+ *
+ * Precondition: {@code cursorSendLoop} must be {@code null}. A live loop's
+ * {@code externalFsnBase} is a construction-time snapshot -- it is never updated
+ * on an already-built loop -- so rolling while one is attached would silently
+ * desynchronize the two: {@link #getAckedFsn()} / {@link #flushAndGetSequence()}
+ * would report post-roll values while every {@code SenderProgressHandler} advance
+ * and {@link SenderError} span the loop emits would stay pinned at pre-roll
+ * values. The recycle swap must call this strictly between tearing the old loop
+ * down and constructing the new one.
+ */
+ private void rollFsnEpochBase(long lastPublishedFsn) {
+ if (cursorSendLoop != null) {
+ throw new IllegalStateException("rollFsnEpochBase must run while cursorSendLoop"
+ + " is null -- the loop's externalFsnBase is a construction-time snapshot,"
+ + " never updated on a live loop; roll strictly between tearing the old"
+ + " loop down and building the new one");
+ }
+ fsnEpochBase += lastPublishedFsn + 1L;
+ }
+
+ /**
+ * Total binary frames whose ACKs have been received and applied since this
+ * sender started. Sender-lifetime: carried across symbol-dictionary
+ * recycles (the rebuilt I/O loop adopts the same counters) and retained
+ * after {@link #close()}.
*/
public long getTotalAcks() {
- CursorWebSocketSendLoop l = cursorSendLoop;
- return l == null ? 0L : l.getTotalAcks();
+ return counters.acks.get();
}
/**
@@ -2199,12 +2699,13 @@ public long getTotalBackgroundDrainersSucceeded() {
/**
* Cumulative number of times {@code appendBlocking} hit a full engine
* ring and parked waiting for the segment manager or the wire to free
- * space. One increment per blocking call, not per spin. Returns 0
- * when the cursor engine has not been allocated yet.
+ * space, since this sender started. One increment per blocking call, not
+ * per spin. Sender-lifetime: carried across symbol-dictionary recycles
+ * (every attached engine adopts the same counters) and retained after
+ * {@link #close()}, where it used to read 0 once the engine was released.
*/
public long getTotalBackpressureStalls() {
- CursorSendEngine e = cursorEngine;
- return e == null ? 0L : e.getTotalBackpressureStalls();
+ return counters.backpressureStalls.get();
}
/**
@@ -2229,47 +2730,55 @@ public long getTotalErrorNotificationsDelivered() {
}
/**
- * Cumulative count of frames re-sent during post-reconnect catch-up
- * windows. Zero in steady state; a sustained nonzero rate signals
- * flapping where every reconnect replays meaningful work.
+ * Count of frames re-sent during post-reconnect catch-up windows since
+ * this sender started. Zero in steady state; a sustained nonzero rate
+ * signals flapping where every reconnect replays meaningful work.
+ * Sender-lifetime: carried across symbol-dictionary recycles and retained
+ * after {@link #close()}.
*/
public long getTotalFramesReplayed() {
- CursorWebSocketSendLoop l = cursorSendLoop;
- return l == null ? 0L : l.getTotalFramesReplayed();
+ return counters.framesReplayed.get();
}
/**
- * Total binary frames the cursor I/O loop has issued to the wire.
+ * Binary frames the cursor I/O loop has issued to the wire since this
+ * sender started, replays included. Sender-lifetime: carried across
+ * symbol-dictionary recycles and retained after {@link #close()}.
*/
public long getTotalFramesSent() {
- CursorWebSocketSendLoop l = cursorSendLoop;
- return l == null ? 0L : l.getTotalFramesSent();
+ return counters.framesSent.get();
}
/**
- * Number of reconnect attempts the cursor I/O loop has issued —
- * succeeded plus failed. Diverges from {@link #getTotalReconnectsSucceeded}
- * when the server is flapping. Returns 0 if no I/O loop is running.
+ * Number of reconnect attempts the cursor I/O loop has issued since this
+ * sender started -- succeeded plus failed. Diverges from
+ * {@link #getTotalReconnectsSucceeded} when the server is flapping.
+ * Sender-lifetime: carried across symbol-dictionary recycles and retained
+ * after {@link #close()}. A recycle's own reconnect runs on the I/O loop's
+ * asynchronous connect path, whose first attempt counts here, so every
+ * completed recycle adds at least one even with no outage.
*/
public long getTotalReconnectAttempts() {
- CursorWebSocketSendLoop l = cursorSendLoop;
- return l == null ? 0L : l.getTotalReconnectAttempts();
+ return counters.reconnectAttempts.get();
}
/**
- * Number of successful reconnects. Returns 0 if no I/O loop is running.
+ * Number of successful reconnects since this sender started.
+ * Sender-lifetime: carried across symbol-dictionary recycles and retained
+ * after {@link #close()}. A recycle's own reconnect counts as one, so every
+ * completed recycle adds at least one even with no outage.
*/
public long getTotalReconnectsSucceeded() {
- CursorWebSocketSendLoop l = cursorSendLoop;
- return l == null ? 0L : l.getTotalReconnects();
+ return counters.reconnects.get();
}
/**
- * Total errors observed by the I/O loop (retriable and terminal combined).
+ * Errors the I/O loop has observed since this sender started (retriable
+ * and terminal combined). Sender-lifetime: carried across symbol-dictionary
+ * recycles and retained after {@link #close()}.
*/
public long getTotalServerErrors() {
- CursorWebSocketSendLoop l = cursorSendLoop;
- return l == null ? 0L : l.getTotalServerErrors();
+ return counters.serverErrors.get();
}
/**
@@ -2578,6 +3087,33 @@ public void reset() {
cachedTimestampNanosColumn = null;
}
+ /**
+ * Advisory request to start a fresh symbol-dictionary epoch. Sets
+ * {@link #manualResetRequested}; if no flush is in flight
+ * ({@code pendingRowCount == 0} -- a first row may still be under
+ * construction; arming is harmless there because the recycle trigger
+ * itself refuses to run mid-row), re-evaluates arming immediately so a
+ * caller that requests a reset between batches does not have to wait for
+ * a later flush to observe {@code isResetArmed()}. A request made
+ * mid-batch is picked up by the next
+ * {@code resetTableBuffersAfterFlush} instead.
+ *
+ * A permanent no-op while {@code symbol_dict_reset} is off: arming gates
+ * on that knob, so a sender configured with the recycle disabled never
+ * acts on the request, however many times it is made. The request is
+ * likewise a permanent no-op on senders that cannot recycle -- ones
+ * without an engine rebuild factory (every {@code connect()}-built
+ * sender) or running on an engine they do not own -- which never arm.
+ */
+ @Override
+ public void resetSymbolDictionary() {
+ checkNotClosed();
+ manualResetRequested = true;
+ if (pendingRowCount == 0) {
+ armIfEligible();
+ }
+ }
+
/**
* Register an async listener for connection-state transitions: initial
* connect, primary failover, endpoint attempt failures, the full address
@@ -2659,10 +3195,18 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) {
seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict());
}
if (engine != null) {
+ engine.adoptCounters(counters);
engine.setSlotLockReleaseListener(this::onSlotLockReleased);
}
}
+ /**
+ * Injects a loop for tests. Deliberately does NOT adopt the sender's
+ * counters: callers start the loop before injecting it, which
+ * {@link CursorWebSocketSendLoop#adoptCounters} refuses, and none of them
+ * reads a {@code getTotal*} accessor -- the sender-side counters simply
+ * never see this loop's counts.
+ */
@TestOnly
public void setCursorSendLoopForTesting(CursorWebSocketSendLoop loop) {
cursorSendLoop = loop;
@@ -2709,6 +3253,43 @@ public synchronized void setDrainerListener(BackgroundDrainerListener listener)
}
}
+ /**
+ * Installs the positive witness {@link #awaitDeferredEngineClose} runs
+ * once it actually begins parking, so a test can prove the await engaged
+ * instead of completing inline -- see
+ * {@code SymbolDictRecycleDeferredCloseTest}.
+ */
+ @TestOnly
+ public void setDeferredCloseParkWitnessForTesting(Runnable witness) {
+ this.deferredCloseParkWitness = witness;
+ }
+
+ /**
+ * Installs a witness {@link #getAckedFsn()} runs between its two volatile
+ * reads (epoch base first, engine second), so a test can hold a monitor
+ * thread on the torn (old base, fresh engine) pair across a real recycle.
+ */
+ @TestOnly
+ public void setAckedFsnReadWitnessForTesting(Runnable witness) {
+ this.ackedFsnReadWitness = witness;
+ }
+
+ /**
+ * Installs the factory a symbol-dictionary recycle uses to rebuild its
+ * cursor engine on the emptied slot ({@code Sender.build()} installs the
+ * builder's). {@code null} stops future arming ({@link #armIfEligible()}
+ * never arms a sender that cannot rebuild) but does not cancel a recycle
+ * that is already armed or pending, so it must not be cleared while one
+ * is. May be called before or after connect; takes effect at the next
+ * flush-tail arming check.
+ *
+ * @throws LineSenderException if the sender is closed
+ */
+ public void setEngineRebuildFactory(EngineRebuildFactory factory) {
+ checkNotClosed();
+ this.engineRebuildFactory = factory;
+ }
+
/**
* Configure the user-supplied error handler. May be called either before
* or after {@code connect()} — when called after, the change propagates
@@ -2738,6 +3319,16 @@ public void setErrorInboxCapacity(int capacity) {
this.errorInboxCapacity = capacity;
}
+ @TestOnly
+ public void setLoopStartFaultForTesting(Runnable fault) {
+ this.loopStartFault = fault;
+ }
+
+ @TestOnly
+ public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) {
+ this.recycleDeferredCloseMaxWaitMillis = millis;
+ }
+
public void setTransactional(boolean transactional) {
this.transactional = transactional;
}
@@ -2920,6 +3511,12 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) {
@Override
public QwpWebSocketSender table(CharSequence tableName) {
checkNotClosed();
+ checkRecycleFailure();
+ if (recycleResume != RecycleResume.NONE) {
+ resumeRecycleIfPending();
+ } else if (resetArmed) {
+ maybeRecycleForDictReset();
+ }
// Fast path: if table name matches current, skip hashmap lookup
if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) {
return this;
@@ -3017,15 +3614,19 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi)
/**
* True iff this sender has at least once installed a live (connected
* + upgraded) WebSocket. Sticky — once true, stays true even after a
- * subsequent disconnect. Lets a {@link SenderErrorHandler}
- * disambiguate a "never reached the server" terminal failure (likely
- * a config typo or firewall block) from a "lost connection after we
- * were up" failure (likely transient). Returns {@code false} if no
- * I/O loop is running.
+ * subsequent disconnect, including through a symbol-dict recycle's
+ * loop-null window (mid-swap, or after a failed reconnect setup).
+ * Lets a {@link SenderErrorHandler} disambiguate a "never reached the
+ * server" terminal failure (likely a config typo or firewall block)
+ * from a "lost connection after we were up" failure (likely
+ * transient). Returns {@code false} only if no loop has ever
+ * connected in this sender's lifetime.
*/
public boolean wasEverConnected() {
+ // Sticky by contract: fall back to the sender-lifetime flag while no
+ // loop is installed (mid-recycle, or after a failed reconnect setup).
CursorWebSocketSendLoop l = cursorSendLoop;
- return l != null && l.hasEverConnected();
+ return hasLoopEverConnected || (l != null && l.hasEverConnected());
}
private static Throwable captureCloseError(Throwable terminalError, Throwable t) {
@@ -3674,6 +4275,35 @@ private void checkNotClosed() {
checkConnectionError();
}
+ /**
+ * Terminal latch for the one symbol-dictionary recycle failure that is not
+ * resumable: a rebuilt engine that recovered UNACKED frames from the slot
+ * the outgoing engine's fully-drained close was supposed to have emptied
+ * (see {@link #completeRecycleRebuild}). That proves the everything-acked
+ * barrier the swap rests on was breached, so the producer's fresh
+ * dictionary and the slot's on-disk state have diverged and this sender
+ * refuses further use. Every OTHER recycle failure -- a wedged SF worker,
+ * an interrupted producer thread, a momentary rebuild fault, a
+ * post-cleanup fsync warning, a failed step-7 reconnect -- is transient:
+ * it throws to the triggering caller, leaves the counter un-bumped and
+ * the recycle pending ({@link #recycleResume}), and the next send
+ * finishes the swap. Checked by
+ * {@link #table(CharSequence)}, the flush-family
+ * entry points ({@link #flush()}, {@link #flushAndGetSequence()},
+ * {@link #drain(long)}, {@link #awaitAckedFsn(long, long)}), and
+ * {@code sendRow()} (closing the fluent-chain corner where a caller
+ * continues {@code .symbol(...).atNow()} against a {@code currentTableBuffer}
+ * selected before the latch, without an intervening {@code table()} call)
+ * -- deliberately NOT by {@link #close()}, which must still be able to
+ * tear down a latched sender.
+ */
+ private void checkRecycleFailure() {
+ if (recycleFailure != null) {
+ throw new LineSenderException(recycleFailure)
+ .put("sender is terminal: symbol dictionary recycle failed");
+ }
+ }
+
private void checkTableSelected() {
if (currentTableBuffer == null) {
throw new LineSenderException("table() must be called before adding columns");
@@ -3727,7 +4357,11 @@ private synchronized Throwable closeRemainingResources(Throwable terminalError)
slotLockReleased = false;
retainedEngine = engine;
}
- } else {
+ } else if (retainedEngine == null) {
+ // No engine and nothing retained: no flock left to report. A
+ // non-null retainedEngine (a recycle's deferred-close await
+ // timed out) still holds the slot flock, so leave the flag
+ // false and let isSlotLockReleased() re-probe it.
slotLockReleased = true;
}
if (errorDispatcher != null) {
@@ -3899,10 +4533,12 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn
long boundary = Math.max(lastCommitBoundaryFsn, cursorEngine.recoveredCommitBoundaryFsn());
long target = Math.min(published, boundary);
if (target < published) {
+ long externalTarget = fsnEpochBase + target;
+ long externalPublished = fsnEpochBase + published;
LOG.warn("close() abandoning {} uncommitted deferred frame(s) [commitBoundaryFsn={}, publishedFsn={}] "
+ "-- their transaction was never committed; the server rolls their rows back. "
+ "Call flush() before close() to commit, or ignore if the abort is intentional.",
- published - target, target, published);
+ published - target, externalTarget, externalPublished);
}
if (cursorEngine.ackedFsn() >= target) {
return;
@@ -3939,6 +4575,8 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn
}
if (System.nanoTime() >= deadlineNanos) {
long acked = cursorEngine.ackedFsn();
+ long externalTarget = fsnEpochBase + target;
+ long externalAcked = fsnEpochBase + acked;
// Name the outage the I/O thread is riding out, when there is one. A
// foreground sender now retries endpoint-policy rejections indefinitely,
// so a revoked token reaches the operator HERE, and blaming timeout
@@ -3946,11 +4584,11 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn
CursorWebSocketSendLoop loop = cursorSendLoop;
Throwable outage = loop == null ? null : loop.lastReconnectError();
LOG.warn("close() drain timed out after {}ms [target={} acked={}], pending data may be lost{}",
- closeFlushTimeoutMillis, target, acked,
+ closeFlushTimeoutMillis, externalTarget, externalAcked,
outage == null ? "" : "; wire is not draining: " + outage.getMessage());
throw new LineSenderException("close() drain timed out after ")
.put(closeFlushTimeoutMillis).put(" ms [targetFsn=")
- .put(target).put(", ackedFsn=").put(acked)
+ .put(externalTarget).put(", ackedFsn=").put(externalAcked)
.put("] - server did not acknowledge ")
.put(target - acked)
.put(outage == null
@@ -4002,6 +4640,7 @@ private void ensureActiveBufferReady() {
private void ensureConnected() {
checkNotClosed();
+ resumeRecycleIfPending();
if (connected) {
return;
}
@@ -4018,7 +4657,14 @@ private void ensureConnected() {
connectionListener, connectionListenerInboxCapacity);
}
CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory();
- switch (initialConnectMode) {
+ // initialConnectMode is an *initialization* policy. After the first
+ // successful connect the SF contract forbids foreground connects on
+ // the producer thread, so re-entries (the recycle's step 7, or its
+ // retry after a failed loop start) always defer to the I/O thread.
+ Sender.InitialConnectMode effectiveMode = hasInitialConnectRun
+ ? Sender.InitialConnectMode.ASYNC
+ : initialConnectMode;
+ switch (effectiveMode) {
case SYNC:
client = CursorWebSocketSendLoop.connectWithRetry(
reconnectFactory,
@@ -4035,10 +4681,13 @@ private void ensureConnected() {
// connect commit to V1 because cursor segments are immutable;
// a future version bump must account for that. Transport
// failures retry indefinitely on the I/O thread (Invariant B).
- // But a terminal auth, upgrade or capability rejection on this
- // initial connect -- before the wire is ever up -- is surfaced
- // to the async SenderErrorHandler and latched for a close()
- // rethrow, not retried.
+ // But a terminal auth, upgrade or capability rejection on the
+ // INITIAL deferred connect -- before the wire is ever up -- is
+ // surfaced to the async SenderErrorHandler and latched for a
+ // close() rethrow, not retried. A re-entry after a prior
+ // connect (the recycle's step 7) seeds the fresh loop with
+ // markEverConnected(), so the same rejection there is retried
+ // like any post-connect failure.
client = null;
break;
case OFF:
@@ -4070,7 +4719,12 @@ private void ensureConnected() {
maxFrameRejections,
poisonMinEscalationWindowMillis,
catchUpCapGapMinEscalationWindowMillis,
- CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
+ CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND,
+ fsnEpochBase);
+ // Sender-lifetime counters: adopted before start(), like the
+ // dispatchers, so the fresh loop keeps counting where the recycle's
+ // outgoing loop stopped.
+ cursorSendLoop.adoptCounters(counters);
// Plug the async-delivery sink before start() so the I/O thread
// never observes a null dispatcher between recordFatal and
// notification — the test for null in dispatchError handles
@@ -4094,6 +4748,19 @@ private void ensureConnected() {
// the loop no longer fires a terminal budget-exhaustion event -- it
// retries indefinitely.)
cursorSendLoop.setConnectionDispatcher(connectionDispatcher);
+ // Seed the fresh loop's own hasEverConnected before it can observe
+ // any endpoint-policy failure: without this, a symbol-dict
+ // recycle's rebuilt loop starts believing it has never connected
+ // (ASYNC startup always hands the constructor a null client),
+ // which would wrongly re-arm endpointPolicyFailureIsTerminal()'s
+ // startup-terminal branch for a FOREGROUND sender that already
+ // reached the server in a prior loop instance.
+ if (hasLoopEverConnected) {
+ cursorSendLoop.markEverConnected();
+ }
+ if (loopStartFault != null) {
+ loopStartFault.run();
+ }
cursorSendLoop.start();
} catch (Throwable t) {
// start() (or dispatcher construction) failed after cursorSendLoop was
@@ -4132,24 +4799,36 @@ private void ensureConnected() {
// client; same path runs on every reconnect.
LOG.info("Connected to WebSocket [host={}, port={}, qwpVersion={}, serverMaxBatchSize={}, effectiveAutoFlushBytes={}]",
host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes);
+ hasLoopEverConnected = true;
} else {
- // Async mode: I/O thread will drive the connect. Encoder uses
- // its default version (V1). The per-batch symbol-dict watermark still
- // gets reset for consistency with the sync path; the post-connect
- // replay path needs no producer-side reset signal (see below).
+ // Deferred connect: the I/O thread will drive it, on the sender's
+ // true initial connect (hasInitialConnectRun still false here) or
+ // on a post-initial re-entry such as the recycle's step 7. Either
+ // way the encoder keeps whatever version was already negotiated
+ // (V1 -- the only supported wire version today); a re-entry never
+ // resets it. The per-batch symbol-dict watermark still gets reset
+ // for consistency with the sync path; the post-connect replay
+ // path needs no producer-side reset signal (see below).
Endpoint ep = endpoints.get(0);
- LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]",
- ep.host, ep.port, endpoints.size());
+ if (hasInitialConnectRun) {
+ LOG.info("Reconnect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]",
+ ep.host, ep.port, endpoints.size());
+ } else {
+ LOG.info("Initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]",
+ ep.host, ep.port, endpoints.size());
+ }
}
// Server starts fresh on each connection, so reset the per-batch
- // symbol-dict watermark. Every frame still carries its full inline schema,
- // and the fresh server's dictionary is re-established either by a full-dict
- // frame (full-dict mode) or by an I/O-thread catch-up frame before replay
- // (delta mode), so post-reconnect replay needs no producer-side reset signal.
+ // symbol-dict watermark when nothing is staged against it. Every frame
+ // still carries its full inline schema, and the fresh server's dictionary
+ // is re-established either by a full-dict frame (full-dict mode) or by an
+ // I/O-thread catch-up frame before replay (delta mode), so post-reconnect
+ // replay needs no producer-side reset signal.
resetSymbolDictStateForNewConnection();
connectionError.set(null);
connected = true;
+ hasInitialConnectRun = true;
}
private void ensureNoInProgressRow() {
@@ -4538,6 +5217,705 @@ private void resetTableBuffersAfterFlush() {
currentTableBufferSnapshotBytes = 0;
pendingRowCount = 0;
firstPendingRowTimeNanos = 0;
+ armIfEligible();
+ }
+
+ /**
+ * Re-evaluates whether the symbol-dictionary recycle should be armed:
+ * {@code resetEnabled} is on, the sender can actually rebuild ({@link
+ * #engineRebuildFactory} is set and {@link #ownsCursorEngine}), AND
+ * either the global dictionary has reached the effective bar
+ * {@code max(resetThresholdSymbols, resetFloorSymbols)} distinct entries
+ * or a caller requested a reset via {@link #resetSymbolDictionary()}.
+ * Deliberately ignores {@code deltaDictEnabled} -- a producer degraded to
+ * full self-sufficient frames still benefits from bounding its dictionary
+ * size, and a manual request is honoured regardless of mode.
+ *
+ * A sender that cannot rebuild -- no {@link #engineRebuildFactory} (every
+ * public {@code QwpWebSocketSender.connect(...)} overload leaves it null
+ * -- only {@code Sender.build()} installs one), or a cursor engine this
+ * sender does not own ({@code setCursorEngine(engine, false)}'s contract:
+ * the caller retains ownership, so closing it out from under them would
+ * be a use-after-free from the caller's point of view) -- must never arm.
+ * Since the recycle feature is default-on and {@code
+ * resetSymbolDictionary()} is a public advisory API, arming a sender with
+ * no way to ever act on the request would leave {@code isResetArmed()}
+ * reading true forever alongside a permanently-0 epoch counter,
+ * misleading monitoring.
+ *
+ * Called from two safe points only: the tail of
+ * {@link #resetTableBuffersAfterFlush()} (no row in progress, this flush's
+ * data already handed to the engine) and {@link #resetSymbolDictionary()}
+ * when nothing is in flight ({@code pendingRowCount == 0}). Never from the
+ * per-symbol registration path ({@link #getOrAddGlobalSymbol}) -- arming
+ * mid-row or mid-encode would observe a dictionary size that has not yet
+ * settled for this batch.
+ *
+ * Arming is only ever triggered from {@code table(CharSequence)}'s
+ * row-start hook: a producer that stops calling {@code table()}, or whose
+ * {@code table()} calls never observe a drained ring, stays armed
+ * indefinitely -- by design; see {@link Sender#resetSymbolDictionary()}'s
+ * documented trigger contract. A pending resume ({@link #resumeRecycleIfPending()})
+ * may still complete an already-triggered recycle from a later
+ * {@code flush()}-driven {@code ensureConnected()} call, with no fresh
+ * {@code table()} call involved.
+ */
+ private void armIfEligible() {
+ boolean shouldArm = resetEnabled
+ && engineRebuildFactory != null
+ && ownsCursorEngine
+ && (globalSymbolDictionary.size() >= Math.max(resetThresholdSymbols, resetFloorSymbols)
+ || manualResetRequested);
+ if (shouldArm && !resetArmed) {
+ armedSinceNanos = System.nanoTime();
+ starvationWaitDoneThisArm = false;
+ }
+ resetArmed = shouldArm;
+ }
+
+ /**
+ * Recycle step 3's deferred-close await. A fully-drained engine close
+ * normally completes inline ({@code isCloseCompleted()} true on return),
+ * making this a single volatile read. When the SF worker was wedged in a
+ * syscall past {@code SegmentManager}'s bounded join, the close instead
+ * returned with the slot flock retained and its release deferred to the
+ * worker's exit path -- exactly the transient disk stall the
+ * deferred-close machinery exists to survive. Park (the same
+ * {@code awaitAckedFsn}-shaped wait the starvation policy uses) until the
+ * deferred cleanup confirms the release; each pass also re-arms the
+ * shared flock-release retry driver for the close-ran-but-release-failed
+ * case, mirroring {@link #isSlotLockReleased()}'s re-probe.
+ *
+ * Exhausting {@link #recycleDeferredCloseMaxWaitMillis} throws; the
+ * recycle stays pending ({@link RecycleResume#REBUILD}) and the next send
+ * probes the close again. The budget is spent once per pending close: a
+ * resume while the worker is still wedged rethrows without parking, so a
+ * stalled worker costs the producer one bounded wait, not one per call.
+ * Before throwing, hand the still-locked engine to {@link #retainedEngine}
+ * so a pool re-probe ({@link #isSlotLockReleased()}) can still recover the
+ * slot's capacity if the worker ever exits.
+ */
+ private void awaitDeferredEngineClose(CursorSendEngine outgoing) {
+ if (outgoing.isCloseCompleted()) {
+ recycleDeferredCloseDeadlineNanos = Long.MIN_VALUE;
+ return;
+ }
+ if (recycleDeferredCloseDeadlineNanos == Long.MIN_VALUE) {
+ LOG.warn("symbol dictionary recycle waiting for a deferred engine close: the SF worker "
+ + "did not quiesce, so the slot lock is still held [maxWaitMillis={}]",
+ recycleDeferredCloseMaxWaitMillis);
+ Runnable witness = deferredCloseParkWitness;
+ if (witness != null) {
+ witness.run();
+ }
+ recycleDeferredCloseDeadlineNanos = System.nanoTime()
+ + recycleDeferredCloseMaxWaitMillis * 1_000_000L;
+ }
+ // Same interrupt policy as maybeBlockForStarvedReset: clear per park
+ // iteration, restore on ALL throw exits (the deadline throw included
+ // -- LineSenderException is a RuntimeException, so the catch below
+ // covers it too; a stray throw from isCloseCompleted() or
+ // ensureFlockReleaseRetryScheduled() is covered the same way), and
+ // swallow on the completed exit.
+ boolean wasInterrupted = false;
+ try {
+ while (!outgoing.isCloseCompleted()) {
+ outgoing.ensureFlockReleaseRetryScheduled();
+ if (System.nanoTime() >= recycleDeferredCloseDeadlineNanos) {
+ retainedEngine = outgoing;
+ slotLockReleased = false;
+ throw new LineSenderException("symbol dictionary recycle could not yet reclaim "
+ + "its slot: the engine's deferred close did not release the "
+ + "slot lock within " + recycleDeferredCloseMaxWaitMillis
+ + " ms (SF worker stalled); the recycle stays pending and is retried "
+ + "on the next send");
+ }
+ java.util.concurrent.locks.LockSupport.parkNanos(50_000L);
+ wasInterrupted |= Thread.interrupted();
+ }
+ } catch (Error e) {
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ throw e;
+ } catch (RuntimeException e) {
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ throw e;
+ }
+ recycleDeferredCloseDeadlineNanos = Long.MIN_VALUE;
+ }
+
+ private void closeRecoveredEngine(CursorSendEngine recovered) {
+ recyclePendingOutgoing = recovered;
+ try {
+ recovered.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ LOG.warn("recovered engine close reported a failure during the symbol dictionary "
+ + "recycle; deferring to the close-completion probe", t);
+ }
+ awaitDeferredEngineClose(recovered);
+ recyclePendingOutgoing = null;
+ retainedEngine = null;
+ }
+
+ /**
+ * The recycle's tail: await the outgoing engine's (possibly deferred)
+ * close, rebuild a fresh engine on the emptied slot, and only then
+ * commit the swap -- roll the FSN base, install the fresh dictionary,
+ * advance the counter, wire the engine, reconnect. Every phase before
+ * the commit is idempotent, so both {@link #recycleForDictReset()} and a
+ * REBUILD resume run this; a transient throw leaves
+ * {@code recycleResume == REBUILD} for the next attempt. Only a rebuild
+ * that recovered UNACKED frames -- a genuine breach of the barrier's
+ * everything-acked proof -- latches {@link #recycleFailure}.
+ */
+ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
+ CursorSendEngine outgoing = recyclePendingOutgoing;
+ if (outgoing != null) {
+ awaitDeferredEngineClose(outgoing); // throws transient while wedged
+ recyclePendingOutgoing = null;
+ retainedEngine = null;
+ }
+ // The rebuild below takes the slot flock again; a stale true (an
+ // isSlotLockReleased() re-probe of the outgoing engine while the
+ // recycle stayed pending) no longer describes this sender's state.
+ slotLockReleased = false;
+ // Replace the dictionary, don't clear(). Allocated before anything
+ // acquires the slot, so an allocation failure leaves nothing to unwind.
+ GlobalSymbolDictionary fresh = new GlobalSymbolDictionary(Math.max(dictSizeAtSwap, 64));
+ // step 4: rebuild the engine on the now-empty slot.
+ CursorSendEngine rebuilt = rebuildEngineOrAbandon(
+ "symbol dictionary recycle could not rebuild its engine; retried on the next send");
+ if (rebuilt.wasRecoveredFromDisk()) {
+ // The outgoing close's empties-the-slot contract can miss
+ // benignly: a transiently failed segment unlink retains the ack
+ // watermark, and the SF design is that the NEXT engine on the
+ // slot recovers those segments as fully acked and retries the
+ // unlink on its own close. Heal by doing exactly that. Only a
+ // recovery holding UNACKED frames is a genuine breach: latch.
+ if (rebuilt.publishedFsn() > rebuilt.ackedFsn()) {
+ throw latchRecycleBreach(rebuilt, dictSizeAtSwap);
+ }
+ closeRecoveredEngine(rebuilt); // fully drained: retries the segment unlink
+ rebuilt = rebuildEngineOrAbandon(
+ "symbol dictionary recycle could not rebuild its engine after healing "
+ + "leftover acked segments; retried on the next send");
+ if (rebuilt.wasRecoveredFromDisk()) {
+ // Re-check: a breach the first pass could not see (the heal's
+ // close reshaped what recovery finds) must latch here too,
+ // otherwise it loops forever behind a resumable "acked" message.
+ if (rebuilt.publishedFsn() > rebuilt.ackedFsn()) {
+ throw latchRecycleBreach(rebuilt, dictSizeAtSwap);
+ }
+ closeRecoveredEngine(rebuilt);
+ throw new LineSenderException(
+ "symbol dictionary recycle keeps recovering leftover acked segments "
+ + "(slot cleanup not durable yet); retried on the next send");
+ }
+ }
+ // COMMIT (steps 5 + 6): pure producer-side state, and nothing below
+ // can throw. Step 5 rolls the external FSN base past every FSN the
+ // outgoing epoch handed out (the -1 no-publish case adds 0); it must
+ // run with cursorSendLoop == null, which step 2 guarantees and the
+ // step-7 reconnect below only undoes afterwards.
+ rollFsnEpochBase(recyclePendingLastPublishedFsn);
+ globalSymbolDictionary = fresh;
+ sentMaxSymbolId = -1;
+ currentBatchMaxSymbolId = -1;
+ lastCommitBoundaryFsn = -1L;
+ symbolDictEpoch++;
+ resetArmed = false;
+ manualResetRequested = false;
+ // Anti-thrash floor: see resetFloorSymbols. max() keeps "never lowered"
+ // true for the manual valve too: resetSymbolDictionary() bypasses the
+ // size gate in armIfEligible, so its swap can run at a dictionary far
+ // below the floor, and 2 x that size would otherwise re-open the
+ // doubling ladder the floor exists to close.
+ resetFloorSymbols = Math.max(resetFloorSymbols, Math.min(dictSizeAtSwap * 2,
+ QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2));
+ // Deliberately re-derived (not carried over): the healing half
+ // of the recycle contract -- a sender that degraded to full
+ // frames heals back into delta mode once the underlying fault
+ // clears; a persistent fault just degrades the fresh engine
+ // again on its first append (SymbolDictRecycleHealingTest).
+ deltaDictEnabled = rebuilt.isDeltaDictEnabled();
+ rebuilt.adoptCounters(counters);
+ cursorEngine = rebuilt;
+ ownsCursorEngine = true;
+ cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
+ recycleResume = RecycleResume.NONE;
+ recyclePendingLastPublishedFsn = -1L;
+ // step 7: reconnect (the swap has already committed).
+ // hasInitialConnectRun forces ensureConnected's ASYNC branch here, so
+ // the deferred socket connect never parks the producer thread; the
+ // loop retries indefinitely on the I/O thread, and a failed setup
+ // here leaves a coherent, merely-disconnected sender that is retried
+ // by the next sendRow()'s ensureConnected().
+ try {
+ ensureConnected();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ LOG.warn("symbol dictionary swap committed but starting its deferred reconnect "
+ + "failed; sender stays disconnected on the fresh epoch and retries "
+ + "the setup on the next send [epoch={}, dictSizeAtSwap={}]",
+ symbolDictEpoch, dictSizeAtSwap, t);
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put("symbol dictionary recycle reconnect failed");
+ }
+ LOG.info("symbol dictionary recycled [epoch={}, dictSizeAtSwap={}, pauseMicros={}]",
+ symbolDictEpoch, dictSizeAtSwap, (System.nanoTime() - startNanos) / 1000L);
+ }
+
+ /**
+ * True once every FSN this engine has published has also been
+ * server-acknowledged (or nothing has been published yet). The barrier
+ * {@link #recycleForDictReset()} waits for: the swap tears the cursor
+ * engine down, so it must never run while a frame is still in flight.
+ *
+ * Read order matters: {@code publishedFsn} (producer-written, cannot move
+ * during this call -- we ARE the producer) first, then {@code ackedFsn}
+ * (monotone, I/O-thread-written) second. Reading them in the other order
+ * could observe a published advance without its matching ack and falsely
+ * report drained.
+ */
+ private boolean isRingDrained() {
+ long published = cursorEngine.publishedFsn();
+ return published < 0 || cursorEngine.ackedFsn() >= published;
+ }
+
+ /**
+ * The recycle's one non-resumable verdict: a rebuild recovered UNACKED
+ * frames from the slot the outgoing engine's fully-drained close was
+ * supposed to have emptied, so the producer's fresh dictionary and the
+ * slot's on-disk state have genuinely diverged. Latches
+ * {@link #recycleFailure}, disposes the rebuilt engine, and always throws
+ * -- the declared return type only lets callers write
+ * {@code throw latchRecycleBreach(...)}.
+ */
+ private RuntimeException latchRecycleBreach(CursorSendEngine rebuilt, int dictSizeAtSwap) {
+ LineSenderException breach = new LineSenderException(
+ "symbol dictionary recycle rebuilt on a slot holding unacknowledged "
+ + "frames: the outgoing engine's fully-drained close contract "
+ + "was breached");
+ recycleFailure = breach;
+ recycleResume = RecycleResume.NONE;
+ try {
+ rebuilt.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable ignored) {
+ // terminal either way; the retained-engine probe below covers a deferred close
+ }
+ if (!rebuilt.isCloseCompleted()) {
+ retainedEngine = rebuilt;
+ slotLockReleased = false;
+ }
+ LOG.error("symbol dictionary recycle failed; sender is now terminal "
+ + "[epoch={}, dictSizeAtSwap={}]", symbolDictEpoch, dictSizeAtSwap, breach);
+ throw breach;
+ }
+
+ /**
+ * Starvation policy: when the ring is NOT drained at arming time, waits
+ * out an opportunistic window before giving up for this armed window.
+ * Refuses (returns immediately) in three cases: {@code resetMaxWaitMillis
+ * <= 0} (blocking disabled), a wait already ran for this arm cycle
+ * ({@link #starvationWaitDoneThisArm} -- at most one blocking wait per
+ * armed window), or a deferred-commit group is open
+ * ({@link #hasDeferredMessages}). That last guard is a data-safety
+ * requirement, not an optimisation: the server withholds acks for
+ * {@code FLAG_DEFER_COMMIT} frames by design until the closing commit
+ * lands, and this producer thread is the only one that could ever send
+ * that commit -- blocking here would just run out the clock every time,
+ * while starving the caller of the thread it needs to actually close the
+ * group.
+ *
+ * Otherwise waits (parked, {@code awaitAckedFsn}-shaped) until either the
+ * ring drains -- in which case the recycle runs synchronously before
+ * returning -- or {@code resetMaxWaitMillis} elapses from THIS call, in
+ * which case it gives up, counts the timeout, and leaves
+ * {@link #resetArmed} set so a later drained {@link #table(CharSequence)}
+ * call can still recycle opportunistically.
+ */
+ private void maybeBlockForStarvedReset() {
+ if (resetMaxWaitMillis <= 0 || starvationWaitDoneThisArm) {
+ return;
+ }
+ if (hasDeferredMessages) {
+ return;
+ }
+ if (System.nanoTime() - armedSinceNanos < resetMaxWaitMillis * 1_000_000L) {
+ return;
+ }
+ starvationWaitDoneThisArm = true;
+ long deadlineNanos = System.nanoTime() + resetMaxWaitMillis * 1_000_000L;
+ // parkNanos returns immediately while the thread's interrupt flag is
+ // set. Clear the flag each time a park returns so the wait keeps its
+ // time budget instead of busy-spinning; restore it on the timeout and
+ // throw exits only. The drained exit deliberately swallows it (the
+ // flock-release retry driver's policy): a restored flag would make
+ // recycleForDictReset()'s loop-close join observe the interrupt and
+ // abandon the recycle this wait just earned.
+ boolean wasInterrupted = false;
+ try {
+ while (!isRingDrained()) {
+ cursorEngine.checkDurability();
+ if (cursorSendLoop != null) {
+ cursorSendLoop.checkError();
+ }
+ checkConnectionError();
+ if (System.nanoTime() >= deadlineNanos) {
+ symbolDictResetStarvationTimeouts++;
+ LOG.warn("symbol dictionary reset starved: backlog not drained within {} ms; "
+ + "staying armed", resetMaxWaitMillis);
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ return;
+ }
+ java.util.concurrent.locks.LockSupport.parkNanos(50_000L);
+ wasInterrupted |= Thread.interrupted();
+ }
+ } catch (Error e) {
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ throw e;
+ } catch (RuntimeException e) {
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ throw e;
+ }
+ recycleForDictReset();
+ }
+
+ /**
+ * Evaluates whether the barrier in {@link #table(CharSequence)} may run
+ * the symbol-dictionary recycle right now. Only ever called with
+ * {@link #resetArmed} true -- {@link #armIfEligible()} already refused to
+ * arm a sender that cannot rebuild, so this method only has to weigh
+ * producer-side state.
+ *
+ * Refuses when there is producer-side state the swap cannot safely tear
+ * down: no connection yet (a V4 sender that has never sent is never
+ * pre-connected here), a flush in flight ({@code pendingRowCount != 0}),
+ * or a row under construction. Otherwise proceeds to the ring-drained
+ * check: if the backlog is empty, recycle immediately; if not, defer to
+ * {@link #maybeBlockForStarvedReset()}'s starvation-wait policy instead
+ * of blocking the caller indefinitely here.
+ */
+ private void maybeRecycleForDictReset() {
+ if (!connected
+ || pendingRowCount != 0
+ || (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
+ return;
+ }
+ if (isRingDrained()) {
+ recycleForDictReset();
+ } else {
+ maybeBlockForStarvedReset();
+ }
+ }
+
+ private CursorSendEngine rebuildEngineOrAbandon(String message) {
+ try {
+ return engineRebuildFactory.rebuild(userErrorHandler());
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ throw new LineSenderException(t).put(message);
+ }
+ }
+
+ /**
+ * The symbol-dictionary recycle swap. Runs synchronously on the producer
+ * thread from the {@link #table(CharSequence)} barrier, once
+ * {@link #maybeRecycleForDictReset()} has proven the ring is drained.
+ * Seven steps, strictly ordered:
+ *
+ * Step 3 mirrors {@code close()}'s deferred-close discipline: when the
+ * outgoing engine's close could not confirm SF-worker quiescence it
+ * returns with the slot flock retained and {@code isCloseCompleted()}
+ * false, releasing both from the worker's exit path. The tail then awaits
+ * that deferred release (bounded by
+ * {@link #RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS}) before step 4 rebuilds
+ * on the slot -- rebuilding against the retained flock would throw
+ * {@code SlotLockContentionException} for what is usually a transient disk
+ * stall.
+ *
+ * Step 7's failure mode is unchanged: by then the swap has committed
+ * (step 6), so a failed step-7 setup (dispatcher construction, loop
+ * build/start -- environmental, since the socket connect itself is
+ * deferred to the I/O thread) leaves a fully coherent sender that is
+ * merely disconnected: {@code connected == false}, loop and client already
+ * closed and nulled by {@link #ensureConnected()}'s own catch, the fresh
+ * engine attached, and the step-6 epoch counter ({@link #symbolDictEpoch})
+ * correctly left incremented because the swap really did happen. It
+ * rethrows loudly to the triggering caller but the sender stays usable,
+ * and the ordinary
+ * {@code sendRow() -> ensureConnected()} path retries the deferred setup
+ * -- and only that -- on the next send. Nothing can fire a second swap
+ * meanwhile: the fresh dictionary is below threshold,
+ * {@code manualResetRequested} was consumed at step 6, and
+ * {@link #maybeRecycleForDictReset()} requires {@code connected}.
+ */
+ private void recycleForDictReset() {
+ final long lastPublishedFsn = cursorEngine.publishedFsn(); // step 1
+ final int dictSizeAtSwap = globalSymbolDictionary.size();
+ final long startNanos = System.nanoTime();
+ if (lastPublishedFsn >= 0) {
+ // Written before teardown: the monitoring accessors keep
+ // reporting this durable watermark while cursorEngine is null.
+ lastRecycleDurableFsn = fsnEpochBase + lastPublishedFsn;
+ }
+ // step 2: close the loop - joins the I/O thread, closes the client.
+ try {
+ if (cursorSendLoop != null) {
+ cursorSendLoop.close();
+ // Read the sticky AFTER close(): close joins the I/O thread,
+ // so a connect that landed mid-window is final here. This is
+ // the only place an ASYNC-initial sender's connect (observed
+ // only by the I/O thread) reaches hasLoopEverConnected.
+ hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
+ cursorSendLoop = null;
+ }
+ client = null;
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ // close() set the loop's stop flag before throwing, so the loop
+ // is irreversibly dying but its I/O thread may still own the
+ // engine. Neither proceed with the swap nor claim connectivity;
+ // abandon, and let resumeRecycleIfPending() finish the close
+ // (a repeated close() converges once the I/O thread exits).
+ connected = false;
+ recycleResume = RecycleResume.CLOSE_LOOP;
+ LOG.warn("symbol dictionary recycle abandoned: closing the outgoing I/O loop "
+ + "failed; the close is finished on the next send [epoch={}]",
+ symbolDictEpoch, t);
+ throw rethrowRecycleAbandoned(t, "symbol dictionary recycle abandoned while closing "
+ + "the outgoing I/O loop; retried on the next send");
+ }
+ // step 3: fully-drained close of the engine - empties the slot and
+ // unlinks the parent-anchored logical slot lock. From here the old
+ // engine cannot come back, so record the REBUILD resume point BEFORE
+ // anything that can throw.
+ CursorSendEngine outgoing = cursorEngine;
+ cursorEngine = null;
+ connected = false;
+ recycleResume = RecycleResume.REBUILD;
+ recyclePendingOutgoing = outgoing;
+ recyclePendingLastPublishedFsn = lastPublishedFsn;
+ try {
+ outgoing.setSlotLockReleaseListener(null);
+ outgoing.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ // A throw with the terminal cleanup nevertheless completed (the
+ // post-cleanup fsyncDir durability warning) is not a swap
+ // failure -- finishClose's finally released the slot regardless.
+ // awaitDeferredEngineClose() below tells the two apart: it
+ // returns immediately when isCloseCompleted(), parks while the
+ // deferred close is in flight, and throws only on a genuinely
+ // dead worker.
+ LOG.warn("outgoing engine close reported a failure during the symbol dictionary "
+ + "recycle; deferring to the close-completion probe", t);
+ }
+ completeRecycleRebuild(dictSizeAtSwap, startNanos);
+ }
+
+ /**
+ * Advances an abandoned recycle. CLOSE_LOOP finishes killing the old
+ * loop (no swap -- the old engine and dictionary are intact and the
+ * armed recycle re-fires from a later barrier, once the reconnect the
+ * next send drives has restored {@code connected}); it re-registers the
+ * producer's delta baseline onto the ring as deferred dictionary chunks
+ * so the fresh loop's catch-up mirror is rebuilt from the ring, and on a
+ * mid-publish failure clamps that baseline to what reached the ring, see
+ * below. REBUILD completes the await/rebuild/commit tail; because the
+ * commit swaps the dictionary, it refuses while producer state could
+ * carry old-dictionary symbol ids (staged rows or a row in progress) --
+ * the caller's row fails, rolls back, and the next table() resumes
+ * cleanly.
+ */
+ private void resumeRecycleIfPending() {
+ if (recycleResume == RecycleResume.NONE) {
+ return;
+ }
+ if (recycleResume == RecycleResume.CLOSE_LOOP) {
+ try {
+ cursorSendLoop.close(); // re-signals; converges once the I/O thread exits
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ throw rethrowRecycleAbandoned(t, "the outgoing I/O loop is still stopping; "
+ + "retried on the next send");
+ }
+ hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
+ cursorSendLoop = null;
+ client = null;
+ // The dead loop took its catch-up mirror with it, but the ring can
+ // carry what the mirror had: re-register [0..sentMaxSymbolId] as
+ // deferred dictionary chunks plus the commit that closes their
+ // group -- the same shape the chunk fallback ships. The fresh loop
+ // ensureConnected() builds next replays them in order ahead of any
+ // data frame, so the server's dictionary is rebuilt before a row
+ // can reference an old id, and the baseline survives instead of
+ // degrading every later flush to a full re-registration (which a
+ // dictionary over the server batch cap can never ship at all).
+ // recycleResume is cleared BEFORE the publish: a throw below must
+ // degrade (the watermark tracks the ringed coverage), never leave this arm
+ // reachable with a null loop.
+ recycleResume = RecycleResume.NONE;
+ // Snapshot the volatile cap ONCE, as flushPendingRows and sendRow do:
+ // serverMaxBatchSize can drop mid-stream via applyServerBatchSizeLimit,
+ // and re-reading it between the guard and the publish call could size
+ // the chunks against a cap that changed underneath them.
+ int cap = serverMaxBatchSize;
+ if (deltaDictEnabled && cap > 0 && sentMaxSymbolId >= 0) {
+ Runnable commitFault = resumeCommitFaultForTesting;
+ try {
+ publishDictionaryChunks(cap, 0, sentMaxSymbolId);
+ if (commitFault != null) {
+ commitFault.run();
+ }
+ sendCommitMessage();
+ // sentMaxSymbolId is KEPT: the ring now carries what the
+ // dead loop's mirror had.
+ } catch (Error e) {
+ // The watermark is already right whichever half threw:
+ // publishDictionaryChunks clamped it to the chunks that
+ // reached the ring before rethrowing, and an Error from the
+ // commitFault seam or from sendCommitMessage() lands here
+ // AFTER the publish returned with full coverage, so the
+ // watermark == the ring's coverage == what the fresh loop's
+ // mirror will hold. The only debt left is the chunks' open
+ // deferred group when sendCommitMessage itself failed: close
+ // it best-effort (commitOrphanedDictionaryChunks never
+ // throws), exactly as the Throwable arm does; the next data
+ // frame closes it too if this attempt fails. The Error
+ // itself is never swallowed.
+ if (hasDeferredMessages) {
+ commitOrphanedDictionaryChunks(e);
+ }
+ throw e;
+ } catch (Throwable t) {
+ // Cap rejection, seal/buffer-recycle timeout, or
+ // appendBlocking's backpressure deadline (no drainer is
+ // attached here, so a full ring parks until the deadline):
+ // publishDictionaryChunks already clamped the watermark to
+ // the ringed coverage (-1 when nothing reached the ring, so
+ // the next flush re-registers from 0). A failure of
+ // sendCommitMessage itself lands here with full coverage
+ // ringed and the watermark kept; it is NOT covered by
+ // publishDictionaryChunks' internal orphan handling (that
+ // fires only when a chunk publish throws), so close the
+ // deferred group's commit debt here -- an open group would
+ // clamp ackedFsn for the connection's whole life.
+ if (hasDeferredMessages) {
+ commitOrphanedDictionaryChunks(t);
+ }
+ LOG.warn("symbol dictionary re-registration after an abandoned recycle "
+ + "failed; the delta baseline now covers the chunks that reached "
+ + "the ring [epoch={}, sentMaxSymbolId={}]",
+ symbolDictEpoch, sentMaxSymbolId, t);
+ }
+ } else {
+ // Full-dict mode has no cross-batch dictionary state to
+ // preserve, and an unknown server cap cannot size chunks:
+ // drop the baseline; the next flush re-registers from 0.
+ sentMaxSymbolId = -1;
+ }
+ return;
+ }
+ // REBUILD
+ if (pendingRowCount != 0
+ || (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
+ throw new LineSenderException(
+ "a symbol dictionary recycle is completing; finish or cancel the "
+ + "in-progress row and retry");
+ }
+ completeRecycleRebuild(globalSymbolDictionary.size(), System.nanoTime());
+ }
+
+ /**
+ * Always throws; the declared return type exists purely so every caller
+ * can write {@code throw rethrowRecycleAbandoned(...)} and make an
+ * accidental fall-through past an abandoned recycle unrepresentable.
+ */
+ private RuntimeException rethrowRecycleAbandoned(Throwable t, String message) {
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put(message);
}
/**
@@ -4587,6 +5965,11 @@ private void sendCommitMessage() {
lastCommitBoundaryFsn = cursorEngine.publishedFsn();
}
+ private SenderErrorHandler userErrorHandler() {
+ SenderErrorHandler h = errorHandler;
+ return h == DefaultSenderErrorHandler.INSTANCE ? null : h;
+ }
+
/**
* Advances the delta baseline once a frame carrying the current batch's
* symbols has been queued onto the ring. No-op in full-dict mode. Only ever
@@ -4600,8 +5983,10 @@ private void advanceSentMaxSymbolId() {
}
/**
- * Stops emitting delta dictionaries for the rest of this sender's life, after the
- * per-slot {@code .symbol-dict} has proved unwritable.
+ * Stops emitting delta dictionaries for the rest of this epoch, after the
+ * per-slot {@code .symbol-dict} has proved unwritable -- a symbol-dictionary
+ * recycle re-derives {@code deltaDictEnabled} from the fresh engine (the
+ * healing contract; see {@link #recycleForDictReset()}).
*
* The side-file can stop accepting appends mid-run -- a full disk or an exhausted
* quota, where SF's own segments stay writable because they are pre-allocated mmap
@@ -4624,8 +6009,9 @@ private void disableDeltaDict(Throwable cause) {
}
deltaDictEnabled = false;
LOG.warn("symbol dictionary persistence failed; this sender has switched to full "
- + "self-sufficient frames for the rest of its life (bandwidth cost only -- "
- + "no data is at risk, and recovery replays such frames without a side file)",
+ + "self-sufficient frames for the rest of this epoch (bandwidth cost only -- "
+ + "no data is at risk, and recovery replays such frames without a side file; "
+ + "a symbol dictionary recycle re-derives delta mode from the fresh engine)",
cause);
}
@@ -4823,49 +6209,66 @@ private void persistNewSymbolsBeforePublish() {
* whole, in order, and {@code RecoveredFrameAnalysis} folds the chunks' deltas
* before it reaches the data frames.
*
- * The baseline is deliberately NOT persisted into {@code sentMaxSymbolId}: full-dict
- * mode carries no cross-batch dictionary state, so every batch re-registers. That
- * keeps the bandwidth cost full-dict mode already accepts, and keeps each batch
- * independently replayable.
+ * For the full-dict caller the baseline is deliberately NOT persisted into
+ * {@code sentMaxSymbolId}: full-dict mode carries no cross-batch dictionary
+ * state, so every batch re-registers. The delta-mode caller (the CLOSE_LOOP
+ * resume) keeps its baseline itself when the publish completes -- these
+ * chunks re-register exactly the ids that baseline already covers -- and
+ * when it does not, the catch below clamps that baseline to the chunks
+ * that reached the ring before closing their group, so the watermark always
+ * equals the ringed coverage.
*
- * All-or-nothing in both directions. Every entry is validated against the cap
- * BEFORE any chunk is published, so a symbol too large to ship at all throws with
- * nothing on the ring; and the sole caller only reaches here once it has proven
- * the batch's bodies fit an empty delta, so a batch that will be rejected never
- * publishes a chunk either.
+ * Nothing reaches the ring for an unshippable batch: every entry is validated
+ * against the cap BEFORE any chunk is published, so a symbol too large to ship
+ * at all throws with nothing on the ring; and neither caller publishes a chunk
+ * for a batch that will be rejected: the full-dict fallback caller
+ * ({@code flushPendingRows}) has already proven the batch's bodies fit an empty
+ * delta before calling here, and the delta-mode resume caller
+ * ({@code resumeRecycleIfPending}) ships no data frames in the group at all. A
+ * failure mid-publish leaves a prefix of the chunks ringed with their group
+ * closed by the orphan commit.
*/
private void publishDictionaryChunks(int cap, int from, int batchMaxId) {
- assert !deltaDictEnabled;
- // Pass one: prove every entry is shippable on its own, before anything
- // reaches the ring. A symbol wider than the cap cannot be split across
- // frames, so it can never be registered and the batch is unshippable --
- // say that plainly rather than let it surface as an unexplained oversized
- // frame from the chunk loop below.
- for (int id = from; id <= batchMaxId; id++) {
- long soloFrameBytes = (long) QwpConstants.HEADER_SIZE
- + NativeBufferWriter.varintSize(id)
- + NativeBufferWriter.varintSize(1)
- + dictionaryEntryWireBytes(id);
- if (soloFrameBytes > cap) {
- throw new BatchTooLargeForCapException("a single symbol value is too large for the server batch cap")
- .put(" [symbolId=").put(id)
- .put(", frameBytes=").put(soloFrameBytes)
- .put(", serverMaxBatchSize=").put(cap).put(']')
- .put("; a symbol value cannot be split across frames -- shorten it, "
- + "raise the server's maximum batch size, or use a varchar "
- + "column instead of symbol for this data");
- }
- }
- // Pass two publishes. Every chunk is a DEFERRED frame, so from the first
- // successful publish onward this method owns a commit debt: if a later chunk
- // throws (sealAndSwapBuffer's buffer-recycle timeout, or appendBlocking's
- // backpressure deadline when the ring is at sf_max_total_bytes), the chunks
- // already on the ring have no rollback and nothing downstream will close their
- // group. Close it here instead -- see commitOrphanedDictionaryChunks.
+ // Delta-mode callers (the CLOSE_LOOP resume re-registration) may only
+ // ship ids the write-ahead persist already made durable -- guaranteed
+ // by persistNewSymbolsBeforePublish's ordering. The full-dict fallback
+ // caller and the degraded delta->full state (a persist failure flipped
+ // deltaDictEnabled off while pd froze) ship self-sufficient chunk
+ // groups and carry no such contract.
+ assert !deltaDictEnabled || isChunkRangeDurable(batchMaxId);
+ // chunkStart is the first id of the chunk in flight: every id below it
+ // is on the ring, nothing at or above it is. Declared outside the try
+ // so the catch can size the coverage that actually landed.
int chunkStart = from;
long chunkBytes = 0;
boolean anyChunkPublished = false;
try {
+ // Pass one: prove every entry is shippable on its own, before anything
+ // reaches the ring. A symbol wider than the cap cannot be split across
+ // frames, so it can never be registered and the batch is unshippable --
+ // say that plainly rather than let it surface as an unexplained oversized
+ // frame from the chunk loop below.
+ for (int id = from; id <= batchMaxId; id++) {
+ long soloFrameBytes = (long) QwpConstants.HEADER_SIZE
+ + NativeBufferWriter.varintSize(id)
+ + NativeBufferWriter.varintSize(1)
+ + dictionaryEntryWireBytes(id);
+ if (soloFrameBytes > cap) {
+ throw new BatchTooLargeForCapException("a single symbol value is too large for the server batch cap")
+ .put(" [symbolId=").put(id)
+ .put(", frameBytes=").put(soloFrameBytes)
+ .put(", serverMaxBatchSize=").put(cap).put(']')
+ .put("; a symbol value cannot be split across frames -- shorten it, "
+ + "raise the server's maximum batch size, or use a varchar "
+ + "column instead of symbol for this data");
+ }
+ }
+ // Pass two publishes. Every chunk is a DEFERRED frame, so from the first
+ // successful publish onward this method owns a commit debt: if a later chunk
+ // throws (sealAndSwapBuffer's buffer-recycle timeout, or appendBlocking's
+ // backpressure deadline when the ring is at sf_max_total_bytes), the chunks
+ // already on the ring have no rollback and nothing downstream will close their
+ // group. Close it in the catch instead -- see commitOrphanedDictionaryChunks.
for (int id = from; id <= batchMaxId; id++) {
int entryBytes = dictionaryEntryWireBytes(id);
// Size the frame this entry WOULD produce, with the count varint the
@@ -4885,6 +6288,22 @@ private void publishDictionaryChunks(int cap, int from, int batchMaxId) {
}
publishDictionaryChunk(chunkStart, batchMaxId);
} catch (Throwable t) {
+ if (deltaDictEnabled) {
+ // The delta-mode caller (the CLOSE_LOOP resume) re-registers
+ // [0..sentMaxSymbolId] from a watermark the ring may now cover
+ // only partially. Clamp it to the coverage that landed so the
+ // orphan commit below, reclaimUnsentSymbolIds' floor and every
+ // later delta anchor at what the fresh loop's mirror will hold:
+ // above it the replay guard fails the loop with a false "host
+ // crash", below it a ringed id could be reclaimed and rebound.
+ // publishDictionaryChunk cannot throw with its frame already
+ // appended (sealAndSwapBuffer's throw sites precede the append),
+ // so chunkStart - 1 is exact; a pre-flight rejection leaves it
+ // at from - 1 (nothing ringed, the mirror is gone, the next
+ // flush re-registers from 0). Full-dict mode never reads the
+ // watermark as a baseline and never reclaims -- leave it alone.
+ sentMaxSymbolId = Math.min(sentMaxSymbolId, chunkStart - 1);
+ }
if (anyChunkPublished) {
commitOrphanedDictionaryChunks(t);
}
@@ -4896,6 +6315,14 @@ private void publishDictionaryChunks(int cap, int from, int batchMaxId) {
}
}
+ private boolean isChunkRangeDurable(int batchMaxId) {
+ if (cursorEngine == null) {
+ return true;
+ }
+ PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict();
+ return pd == null || pd.size() > batchMaxId;
+ }
+
/**
* Publishes one deferred, table-less frame registering symbol ids
* {@code [startId, endId]}. Mirrors {@link #sendCommitMessage()}'s publish
@@ -4903,6 +6330,10 @@ private void publishDictionaryChunks(int cap, int from, int batchMaxId) {
* batch's data frames.
*/
private void publishDictionaryChunk(int startId, int endId) {
+ Runnable chunkFault = chunkPublishFaultForTesting;
+ if (chunkFault != null) {
+ chunkFault.run();
+ }
encoder.setDeferCommit(true);
// confirmedMaxId = startId - 1 makes beginMessage emit deltaStart = startId,
// deltaCount = endId - startId + 1.
@@ -4973,14 +6404,28 @@ private void reclaimUnsentSymbolIds() {
}
private void resetSymbolDictStateForNewConnection() {
- // Runs on the foreground (initial) connect only -- NOT on the I/O thread's
- // reconnect/failover path. The per-batch watermark is drained state, so
- // clearing it here is harmless. sentMaxSymbolId is deliberately left
+ // Runs on the foreground connect only -- NOT on the I/O thread's
+ // reconnect/failover path. sentMaxSymbolId is deliberately left
// untouched: in delta mode the I/O thread re-registers the whole
// dictionary with a catch-up frame on reconnect, so the producer's
// monotonic baseline must survive the wire boundary; resetting it would
// desync the producer from the I/O thread's sent-dictionary count.
- currentBatchMaxSymbolId = -1;
+ //
+ // currentBatchMaxSymbolId is batch-scoped, not connection-scoped: a
+ // flush ships exactly [sentMaxSymbolId+1 .. currentBatchMaxSymbolId],
+ // so clearing it while a batch already references those ids ships a
+ // delta that OMITS them and puts rows on the wire pointing at symbol
+ // ids the server never received. Clearing it used to be unconditional
+ // and harmless because build() connects before the application can
+ // register anything. That no longer holds: a symbol-dictionary recycle
+ // whose step-7 connect failed defers the connect to the next
+ // sendRow(), which runs after symbol() has registered the ids of the
+ // row being built. Reset only from the drained state the old code
+ // assumed.
+ if (pendingRowCount == 0
+ && (currentTableBuffer == null || !currentTableBuffer.hasInProgressRow())) {
+ currentBatchMaxSymbolId = -1;
+ }
}
/**
@@ -5185,7 +6630,9 @@ private void sealAndSwapBuffer() {
// Surface any I/O thread error first — appendBlocking itself only
// throws on PAYLOAD_TOO_LARGE / backpressure deadline, but the
// I/O loop can have failed independently.
- cursorSendLoop.checkError();
+ if (cursorSendLoop != null) {
+ cursorSendLoop.checkError();
+ }
throw new LineSenderException("cursor SF append failed", t);
}
}
@@ -5195,6 +6642,7 @@ private void sealAndSwapBuffer() {
* Rows buffer until flush (explicit or auto-flush).
*/
private void sendRow() {
+ checkRecycleFailure();
ensureConnected();
// Hard guard: a single row whose bytes exceed the server's wire cap
@@ -5324,6 +6772,24 @@ public Endpoint(String host, int port) {
}
}
+ /**
+ * Rebuilds a fresh {@link CursorSendEngine} on this sender's own slot, going
+ * through the identical construct/quarantine code path
+ * {@link Sender.LineSenderBuilder#build} uses.
+ */
+ public interface EngineRebuildFactory {
+ CursorSendEngine rebuild();
+
+ /**
+ * Rebuild with the sender's current user-supplied error handler ({@code null}
+ * when only the default handler is installed), so a quarantine during the
+ * rebuild reaches a handler installed after {@code build()}.
+ */
+ default CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
+ return rebuild();
+ }
+ }
+
/**
* A constant {@code Authorization} header value. Its identity as a type - not the value it yields - is
* what {@link #hasDynamicCredential()} reads, so the drainer can apply the right terminal policy to a
@@ -5342,6 +6808,15 @@ public String get() {
}
}
+ /**
+ * How far an abandoned symbol-dictionary recycle got, and therefore what
+ * {@link #resumeRecycleIfPending()} must still do. See
+ * {@link #recycleResume}.
+ */
+ private enum RecycleResume {
+ NONE, CLOSE_LOOP, REBUILD
+ }
+
private final class ReconnectSupplier implements CursorWebSocketSendLoop.ReconnectFactory {
/**
* Optional caller-owned liveness gate. {@code null} means this factory
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
index facd872b..0889d1ba 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
@@ -1140,7 +1140,8 @@ public void run() {
maxHeadFrameRejections,
poisonMinEscalationWindowMillis,
catchUpCapGapMinEscalationWindowMillis,
- CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
+ CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN,
+ 0L);
// Without this the loop's ridden-out reports -- above all
// "credential-unavailable", the one endpoint-policy failure an ORPHAN
// loop retries rather than latching -- are dispatched into a null, and
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendCounters.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendCounters.java
new file mode 100644
index 00000000..960b3211
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendCounters.java
@@ -0,0 +1,77 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Sender-lifetime observability counters shared by every
+ * {@link CursorWebSocketSendLoop} generation and every {@link CursorSendEngine}
+ * a {@code QwpWebSocketSender} attaches.
+ *
+ * A symbol-dictionary recycle rebuilds both the loop and the engine; the
+ * counters they report must not restart with them, so the sender owns one
+ * instance for its whole life and hands it to each new loop and engine before
+ * use ({@code adoptCounters}). A loop or engine built standalone (tests,
+ * background drainers) starts on a fresh instance of its own, so its getters
+ * behave exactly as they did when the counters were per-instance fields.
+ *
+ * Every field is incremented atomically from whichever I/O thread (or the
+ * caller thread inside a synchronous {@code start()}) is active, and read by
+ * any monitor thread; each read is one atomic load, so a value sampled across
+ * a recycle is never a torn sum.
+ */
+public final class CursorSendCounters {
+ /** ACK frames received and applied; {@code CursorWebSocketSendLoop#getTotalAcks()}. */
+ public final AtomicLong acks = new AtomicLong();
+ /** Producer appends that hit a full ring and parked; {@code CursorSendEngine#getTotalBackpressureStalls()}. */
+ public final AtomicLong backpressureStalls = new AtomicLong();
+ /** Frames re-sent inside post-reconnect catch-up windows. */
+ public final AtomicLong framesReplayed = new AtomicLong();
+ /** Binary frames issued to the wire, replays included. */
+ public final AtomicLong framesSent = new AtomicLong();
+ /** Reconnect attempts, failed and successful alike. */
+ public final AtomicLong reconnectAttempts = new AtomicLong();
+ /** Successful reconnects. */
+ public final AtomicLong reconnects = new AtomicLong();
+ /** Non-OK / non-DURABLE_ACK frames received from the server, retriable and terminal. */
+ public final AtomicLong serverErrors = new AtomicLong();
+
+ /**
+ * Folds every counter of {@code other} into this instance. Used once, when a
+ * loop or engine that started on its own default instance is handed the
+ * sender's shared one, so anything counted before adoption is carried
+ * rather than dropped.
+ */
+ public void addAll(CursorSendCounters other) {
+ acks.addAndGet(other.acks.get());
+ backpressureStalls.addAndGet(other.backpressureStalls.get());
+ framesReplayed.addAndGet(other.framesReplayed.get());
+ framesSent.addAndGet(other.framesSent.get());
+ reconnectAttempts.addAndGet(other.reconnectAttempts.get());
+ reconnects.addAndGet(other.reconnects.get());
+ serverErrors.addAndGet(other.serverErrors.get());
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index 66a0635e..dffe3b4b 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -84,12 +84,14 @@ public final class CursorSendEngine implements QuietCloseable {
private static volatile ThreadFactory flockReleaseRetryThreadFactory =
DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY;
private final long appendDeadlineNanos;
- // Number of times appendBlocking observed BACKPRESSURE_NO_SPARE on its first
- // ring.appendOrFsn attempt. One increment per blocking-call that had to wait
- // for the manager (or for ACKs) — not one per spin-park. Producer-thread
- // writer; volatile because the user may sample it from any thread.
- private final java.util.concurrent.atomic.AtomicLong backpressureStallCount =
- new java.util.concurrent.atomic.AtomicLong();
+ // Sender-lifetime observability counters; only backpressureStalls is used
+ // here: one increment per blocking appendBlocking call that had to wait
+ // for the manager (or for ACKs), not one per spin-park. Producer-thread
+ // writer; any thread may read it. A fresh instance by default;
+ // QwpWebSocketSender hands every engine it attaches its own shared
+ // instance via adoptCounters(), so a symbol-dictionary recycle's rebuilt
+ // engine keeps counting where the outgoing one stopped.
+ private CursorSendCounters counters = new CursorSendCounters();
// Constructed before an owned manager acquires its native path scratch, so
// callback allocation failure cannot orphan manager resources. A timed-out
// close can then hand it to either manager path without allocating.
@@ -923,7 +925,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) {
}
// First miss → record one stall (not one per spin) and start the
// deadline clock.
- backpressureStallCount.incrementAndGet();
+ counters.backpressureStalls.incrementAndGet();
long deadlineNs = System.nanoTime() + appendDeadlineNanos;
while (true) {
long now = System.nanoTime();
@@ -937,7 +939,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) {
lastBackpressureLogNs = now;
LOG.warn("cursor producer backpressured ({} stalls so far); waiting for I/O or periodic disk sync; "
+ "will throw after {} ms",
- backpressureStallCount.get(), appendDeadlineNanos / 1_000_000L);
+ counters.backpressureStalls.get(), appendDeadlineNanos / 1_000_000L);
}
LockSupport.parkNanos(50_000L); // 50 µs
fsn = ring.appendOrFsn(payloadAddr, payloadLen);
@@ -1349,6 +1351,11 @@ public SlotLock getSlotLockForTesting() {
return slotLock;
}
+ @TestOnly
+ public Runnable getSlotLockReleaseListenerForTesting() {
+ return slotLockReleaseListener;
+ }
+
@TestOnly
public long getSyncIntervalNanosForTesting() {
return syncIntervalNanos;
@@ -1623,6 +1630,32 @@ public void setSlotLockReleaseListener(Runnable listener) {
}
}
+ /**
+ * Replaces this engine's counters with the sender's shared, sender-lifetime
+ * instance, folding this engine's own backpressure-stall count into it
+ * (the other counters are the loop's, and an engine handed between
+ * senders must not carry a previous sender's totals). Producer thread
+ * only, before the first {@link #appendBlocking} on this engine -- the
+ * same attach window {@link #setSlotLockReleaseListener} uses.
+ */
+ public void adoptCounters(CursorSendCounters shared) {
+ if (shared == counters) {
+ return;
+ }
+ shared.backpressureStalls.addAndGet(counters.backpressureStalls.get());
+ counters = shared;
+ }
+
+ /**
+ * The engine's current counters holder -- the DEFAULT instance until
+ * {@link #adoptCounters} replaces it. Test-only seam to observe what
+ * {@link #adoptCounters} folds and what it leaves alone.
+ */
+ @TestOnly
+ public CursorSendCounters getCountersForTesting() {
+ return counters;
+ }
+
/**
* Re-arms the shared terminal retry for an engine whose final watermark
* barrier or confirmed flock release is still pending and no longer
@@ -1749,10 +1782,12 @@ public PersistedSymbolDict getPersistedSymbolDict() {
* Number of times {@link #appendBlocking} hit
* {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and
* had to wait for the segment manager (or for ACKs) to free space.
- * One increment per blocking-call, not per spin-park. Cumulative.
+ * One increment per blocking-call, not per spin-park. Cumulative, and
+ * carried across a symbol-dictionary recycle once the owning sender has
+ * adopted this engine (see {@link #adoptCounters}).
*/
public long getTotalBackpressureStalls() {
- return backpressureStallCount.get();
+ return counters.backpressureStalls.get();
}
/**
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 6643bf03..50a6d7ef 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -295,28 +295,19 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
private final WebSocketResponse response = new WebSocketResponse();
private final ResponseHandler responseHandler = new ResponseHandler();
private final CountDownLatch shutdownLatch = new CountDownLatch(1);
- private final AtomicLong totalAcks = new AtomicLong();
// Counters for observability of the durable-ack path. Both are zero
// when durableAckMode is false.
private final AtomicLong totalDurableAcks = new AtomicLong();
private final AtomicLong totalDurableTrimAdvances = new AtomicLong();
- // Cumulative count of frames the loop has re-sent during post-reconnect
- // catch-up windows. Bumped once per frame on every iteration that
- // observes replayTargetFsn >= 0. A flat zero confirms steady state; a
- // sustained nonzero rate means the connection is flapping and replay
- // is doing real work each cycle.
- private final AtomicLong totalFramesReplayed = new AtomicLong();
- private final AtomicLong totalFramesSent = new AtomicLong();
- // Every iteration of the reconnect loop bumps this — failures and
- // success alike. Diverges from totalReconnects (success-only) when the
- // server is flapping. Useful for "is reconnect making progress?"
- // observability.
- private final AtomicLong totalReconnectAttempts = new AtomicLong();
- private final AtomicLong totalReconnects = new AtomicLong();
- // Total non-OK / non-DURABLE_ACK frames received from the server, classified
- // by category. Includes both retriable and terminal outcomes — i.e. every
- // server-side rejection observed regardless of how the loop reacted.
- private final AtomicLong totalServerErrors = new AtomicLong();
+ // Sender-lifetime observability counters: acks, frames sent/replayed,
+ // reconnect attempts and successes, server errors. A fresh instance per
+ // loop by default; QwpWebSocketSender hands every loop generation its own
+ // shared instance via adoptCounters() before start(), so a symbol-
+ // dictionary recycle's rebuilt loop keeps counting where the outgoing one
+ // stopped. Written only by the I/O thread, read by any monitor thread
+ // through the AtomicLongs. Not final: adoptCounters() replaces it before
+ // the I/O thread exists.
+ private CursorSendCounters counters = new CursorSendCounters();
// Delta symbol dictionary catch-up state (see swapClient).
// ALWAYS active -- in memory mode, in disk mode, and (critically) even when the
// per-slot persisted dictionary failed to open. sentDictCount is this loop's model
@@ -416,6 +407,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
// it is engine.ackedFsn() + 1, so the first replayed frame on the new
// connection is wireSeq=0 and server-side cumulative ACKs still line up.
private long fsnAtZero;
+ // Third coordinate: additive offset applied on top of the engine FSN
+ // (fsnAtZero already folded in) to produce the FSN this loop hands to a
+ // user-visible surface -- the progress dispatcher and every SenderError
+ // [fromFsn,toFsn] span. Fixed for the lifetime of one loop instance: 0
+ // for a loop built directly against a live engine, or the sender's
+ // fsnEpochBase snapshot when a symbol-dict recycle rebuilt the engine and
+ // restarted its internal FSNs at 0. Rule: external = externalFsnBase + raw.
+ private final long externalFsnBase;
// Bounded-await backstop budget for close() (see
// DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS). Overridable via
// setShutdownAwaitTimeoutMillis so tests can exercise the timeout branch
@@ -724,7 +723,7 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
reconnectMaxBackoffMillis, durableAckMode,
durableAckKeepaliveIntervalMillis, maxHeadFrameRejections,
poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis,
- CatchUpCapGapPolicy.RETRY_FOREVER);
+ CatchUpCapGapPolicy.RETRY_FOREVER, 0L);
}
/**
@@ -742,7 +741,8 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
int maxHeadFrameRejections,
long poisonMinEscalationWindowMillis,
long catchUpCapGapMinEscalationWindowMillis,
- CatchUpCapGapPolicy catchUpCapGapPolicy) {
+ CatchUpCapGapPolicy catchUpCapGapPolicy,
+ long externalFsnBase) {
if (maxHeadFrameRejections < 1) {
throw new IllegalArgumentException(
"maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections);
@@ -894,6 +894,7 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
// always outlive their borrower. Any growth copy-on-writes into loop-owned memory
// (ensureSentDictCapacity), and releaseSentDictBytes frees only what the loop owns.
this.fsnAtZero = fsnAtZero;
+ this.externalFsnBase = externalFsnBase;
this.parkNanos = parkNanos;
this.reconnectFactory = reconnectFactory;
this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis;
@@ -935,6 +936,11 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
* establishing its first connection, then retries endpoint-policy failures
* indefinitely after it has been live. An orphan drainer returns such failures
* to its owner so the slot can follow its settle/quarantine policy.
+ *
+ * {@code externalFsnBase} is the additive offset this loop folds into every
+ * user-visible FSN it produces (progress-dispatcher advances and
+ * {@link SenderError} spans) -- see {@link #externalFsnBase}. Pass {@code 0L}
+ * unless the caller is replacing an engine a symbol-dict recycle rebuilt.
*/
public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
long fsnAtZero, long parkNanos,
@@ -946,13 +952,14 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
int maxHeadFrameRejections,
long poisonMinEscalationWindowMillis,
long catchUpCapGapMinEscalationWindowMillis,
- ReconnectPolicy reconnectPolicy) {
+ ReconnectPolicy reconnectPolicy,
+ long externalFsnBase) {
this(client, engine, fsnAtZero, parkNanos, reconnectFactory,
reconnectInitialBackoffMillis,
reconnectMaxBackoffMillis, durableAckMode,
durableAckKeepaliveIntervalMillis, maxHeadFrameRejections,
poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis,
- catchUpPolicyFor(reconnectPolicy));
+ catchUpPolicyFor(reconnectPolicy), externalFsnBase);
}
private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) {
@@ -1465,7 +1472,7 @@ public Throwable getTerminalError() {
}
public long getTotalAcks() {
- return totalAcks.get();
+ return counters.acks.get();
}
/**
@@ -1495,22 +1502,22 @@ public long getTotalDurableTrimAdvances() {
* meaningful work.
*/
public long getTotalFramesReplayed() {
- return totalFramesReplayed.get();
+ return counters.framesReplayed.get();
}
public long getTotalFramesSent() {
- return totalFramesSent.get();
+ return counters.framesSent.get();
}
/**
* Total reconnect attempts (succeeded + failed).
*/
public long getTotalReconnectAttempts() {
- return totalReconnectAttempts.get();
+ return counters.reconnectAttempts.get();
}
public long getTotalReconnects() {
- return totalReconnects.get();
+ return counters.reconnects.get();
}
/**
@@ -1519,7 +1526,7 @@ public long getTotalReconnects() {
* the client classified as a {@link SenderError}.
*/
public long getTotalServerErrors() {
- return totalServerErrors.get();
+ return counters.serverErrors.get();
}
/**
@@ -1538,6 +1545,47 @@ public boolean isRunning() {
return running;
}
+ /**
+ * Called by the sender before {@link #start()} when a prior loop of the
+ * same sender already reached the server: restores Invariant B's
+ * past-initialization classification (see {@link
+ * #endpointPolicyFailureIsTerminal()}) across a symbol-dict recycle's
+ * loop rebuild, where the constructor would otherwise seed a fresh
+ * {@code hasEverConnected = false} for the new loop instance (ASYNC
+ * startup always hands the constructor a null client). Public rather
+ * than package-private only because the owning sender lives in a
+ * different package; it is not part of the public {@code Sender} API.
+ * {@code hasEverConnected} is volatile, so this write needs no extra
+ * synchronization to be visible to the I/O thread -- callers still call
+ * it before {@code start()} so the invariant is established before the
+ * loop can observe any endpoint-policy failure.
+ */
+ public void markEverConnected() {
+ hasEverConnected = true;
+ }
+
+ /**
+ * Replaces this loop's counters with the sender's shared, sender-lifetime
+ * instance, folding anything already counted into it. Must run before
+ * {@link #start()}: the I/O thread reads the field after start()'s
+ * happens-before, and a swap under a running I/O thread could lose
+ * increments. {@code QwpWebSocketSender.ensureConnected} calls this on
+ * every loop generation, which is what keeps the sender's
+ * {@code getTotal*} accessors monotone across a symbol-dictionary recycle.
+ *
+ * @throws IllegalStateException if the loop has already started
+ */
+ public void adoptCounters(CursorSendCounters shared) {
+ if (ioThread != null) {
+ throw new IllegalStateException("adoptCounters must run before start()");
+ }
+ if (shared == counters) {
+ return;
+ }
+ shared.addAll(counters);
+ counters = shared;
+ }
+
/**
* Plug an async-delivery sink for {@link SenderConnectionEvent}
* notifications. Connection events fire from
@@ -1786,7 +1834,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
lastReconnectError = initial;
while (running) {
attempts++;
- totalReconnectAttempts.incrementAndGet();
+ counters.reconnectAttempts.incrementAndGet();
try {
WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation);
if (newClient != null) {
@@ -1812,11 +1860,11 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
break;
}
swapClient(newClient);
- totalReconnects.incrementAndGet();
+ counters.reconnects.incrementAndGet();
long elapsedMs = (System.nanoTime() - outageStartNanos) / 1_000_000L;
LOG.info("cursor I/O loop {} succeeded after {}ms, {} attempts; "
+ "replaying from FSN {}",
- phase, elapsedMs, attempts, fsnAtZero);
+ phase, elapsedMs, attempts, externalFsnBase + fsnAtZero);
return;
}
// A null factory result is an unsuccessful connect state, not a cap-gap
@@ -1855,8 +1903,8 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
LOG.error("terminal upgrade error during {} -- won't retry: {}",
phase, e.getMessage());
}
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
SenderError err = new SenderError(
SenderError.Category.SECURITY_ERROR,
SenderError.Policy.TERMINAL,
@@ -1868,7 +1916,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
null,
System.nanoTime()
);
- totalServerErrors.incrementAndGet();
+ counters.serverErrors.incrementAndGet();
recordFatal(new LineSenderServerException(err));
dispatchError(err);
return;
@@ -1895,8 +1943,8 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
// volatile first-writer-wins latch observed by the owner.
capabilityGapTerminal = e;
}
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
SenderError err = new SenderError(
SenderError.Category.PROTOCOL_VIOLATION,
SenderError.Policy.TERMINAL,
@@ -1908,7 +1956,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
null,
System.nanoTime()
);
- totalServerErrors.incrementAndGet();
+ counters.serverErrors.incrementAndGet();
recordFatal(new LineSenderServerException(err));
dispatchError(err);
return;
@@ -2062,7 +2110,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
* producer stays alive and no data is at risk.
*/
private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category, String message) {
- long fromFsn = engine.ackedFsn() + 1L;
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
dispatchError(new SenderError(
category,
SenderError.Policy.RETRIABLE,
@@ -2070,7 +2118,7 @@ private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category,
message,
SenderError.NO_MESSAGE_SEQUENCE,
fromFsn,
- Math.max(fromFsn, engine.publishedFsn()),
+ Math.max(fromFsn, externalFsnBase + engine.publishedFsn()),
null,
System.nanoTime()
));
@@ -2176,9 +2224,11 @@ private void haltOnPoisonedFrame(String lastRejection, long toFsnHint) {
// the operator at those bytes would misattribute the poison. The
// caller supplies the span end: a NACK names the exact frame, so the
// span is that single frame; a non-orderly close cannot single one
- // out, so it spans to publishedFsn.
- long fromFsn = poisonFsn;
- long toFsn = Math.max(fromFsn, toFsnHint);
+ // out, so it spans to publishedFsn. poisonFsn and toFsnHint are both
+ // raw internal FSNs (fsnAtZero already folded in by the caller where
+ // relevant); rebase both by externalFsnBase here.
+ long fromFsn = externalFsnBase + poisonFsn;
+ long toFsn = Math.max(fromFsn, externalFsnBase + toFsnHint);
String msg = "frame at fsn=" + fromFsn + " rejected " + poisonStrikes
+ " consecutive times with no acceptance at or beyond it -- poisoned frame, replay cannot succeed (last: "
+ lastRejection + ')';
@@ -2193,7 +2243,7 @@ private void haltOnPoisonedFrame(String lastRejection, long toFsnHint) {
null,
System.nanoTime()
);
- totalServerErrors.incrementAndGet();
+ counters.serverErrors.incrementAndGet();
recordFatal(new LineSenderServerException(err));
dispatchError(err);
}
@@ -2202,12 +2252,14 @@ private void haltOnPoisonedFrame(String lastRejection, long toFsnHint) {
* Notify the progress dispatcher that the ack watermark advanced to
* {@code ackedFsn}. Caller must already have observed the advance via
* {@link CursorSendEngine#acknowledge}'s boolean return; this method
- * does no further filtering.
+ * does no further filtering. {@code ackedFsn} is the engine-relative FSN
+ * (fsnAtZero already folded in by the caller); this rebases it by
+ * {@link #externalFsnBase} before it reaches the user-visible dispatcher.
*/
private void dispatchProgress(long ackedFsn) {
SenderProgressDispatcher d = progressDispatcher;
if (d != null) {
- d.offer(ackedFsn);
+ d.offer(externalFsnBase + ackedFsn);
}
}
@@ -3211,7 +3263,7 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr,
}
nextWireSeq++; // this catch-up chunk consumed a wire sequence
lastFrameOrPingNanos = System.nanoTime();
- totalFramesSent.incrementAndGet();
+ counters.framesSent.incrementAndGet();
}
@TestOnly
@@ -3537,9 +3589,9 @@ private boolean trySendOne() {
sendOffset = frameEnd;
long fsnSent = fsnAtZero + nextWireSeq;
nextWireSeq++;
- totalFramesSent.incrementAndGet();
+ counters.framesSent.incrementAndGet();
if (replayTargetFsn >= 0) {
- totalFramesReplayed.incrementAndGet();
+ counters.framesReplayed.incrementAndGet();
if (fsnSent >= replayTargetFsn) {
replayTargetFsn = -1L; // catch-up complete
}
@@ -3880,7 +3932,7 @@ public void onBinaryMessage(long payloadPtr, int payloadLen) {
LOG.warn("server ACK wire seq {} outside sent range [0, {}], clamping",
wireSeq, highestSent);
}
- totalAcks.incrementAndGet();
+ counters.acks.incrementAndGet();
long okFsn = fsnAtZero + capped;
if (okFsn > highestOkFsn) {
highestOkFsn = okFsn;
@@ -4004,8 +4056,8 @@ private void handlePreSendRejection(long wireSeq, byte status,
// protocol-violation close path uses (see onClose above): there
// is no FSN we can attribute the rejection to, so we report
// the unacked range the producer can correlate against.
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
String tableName = response.getTableEntryCount() == 1
? response.getTableName(0)
: null;
@@ -4020,7 +4072,7 @@ private void handlePreSendRejection(long wireSeq, byte status,
tableName,
System.nanoTime()
);
- totalServerErrors.incrementAndGet();
+ counters.serverErrors.incrementAndGet();
if (policy == SenderError.Policy.TERMINAL) {
// Latch the typed terminal error before invoking the handler
// so a synchronous probe of getLastTerminalError() / flush()
@@ -4128,12 +4180,12 @@ private void handleServerRejection(long wireSeq) {
status & 0xFF,
response.getErrorMessage(),
wireSeq,
- fsn,
- fsn,
+ externalFsnBase + fsn,
+ externalFsnBase + fsn,
tableName,
System.nanoTime()
);
- totalServerErrors.incrementAndGet();
+ counters.serverErrors.incrementAndGet();
if (policy == SenderError.Policy.TERMINAL) {
// Terminal: stash the typed payload BEFORE dispatching to the
@@ -4156,7 +4208,7 @@ private void handleServerRejection(long wireSeq) {
// no ack progress escalates to a poisoned-frame terminal instead
// of reconnect-looping forever.
LOG.warn("server rejected wire seq {} (category={}, policy={}, status=0x{}) -- recycling connection, will replay from fsn {}",
- wireSeq, category, policy, Integer.toHexString(status & 0xFF), engine.ackedFsn() + 1L);
+ wireSeq, category, policy, Integer.toHexString(status & 0xFF), externalFsnBase + engine.ackedFsn() + 1L);
dispatchError(err);
LineSenderException recycleCause = new LineSenderException(
"server NACK (" + category + ", " + policy + "): "
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
index 074cf0e3..0fa7a645 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
@@ -92,8 +92,17 @@ public final class QwpConstants {
*
* NOT the result-direction cap: {@code QwpResultBatchDecoder.MAX_CONN_DICT_SIZE}
* (8,388,608) governs server-to-client result batches and is unrelated.
- */
- public static final int MAX_SYMBOL_DICTIONARY_SIZE = 1_000_000;
+ *
+ * Compatibility: servers released before QuestDB 10.0.0 cap their
+ * dictionary at 1,000,000, and QWP has no wire-level negotiation of the
+ * limit -- a dictionary this client lets grow past 1M is rejected by
+ * those servers as a terminal parse error. Reachable on defaults: each
+ * recycle raises the re-arm bar to twice the dictionary size at the
+ * swap, capped at half of this constant, so an unbounded-cardinality
+ * producer's dictionary grows to 1M entries per epoch. Only 10.0.0+
+ * servers are supported.
+ */
+ public static final int MAX_SYMBOL_DICTIONARY_SIZE = 2_000_000;
/**
* Maximum table name length in bytes. Mirrors the server's same-named
* constant; used by the decoder to reject malformed wire bytes.
diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
index c9529a13..b9725608 100644
--- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
+++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
@@ -90,6 +90,9 @@ public final class ConfigSchema {
str("sf_max_segment_bytes", Side.INGRESS);
str("sf_max_total_bytes", Side.INGRESS);
str("sf_sync_interval_millis", Side.INGRESS);
+ str("symbol_dict_reset", Side.INGRESS);
+ str("symbol_dict_reset_max_wait_millis", Side.INGRESS);
+ str("symbol_dict_reset_threshold", Side.INGRESS);
str("transaction", Side.INGRESS);
// EGRESS -- the QwpQueryClient applies. Typed where there is a range or
diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java
index 7b4e5f80..095f64ab 100644
--- a/core/src/main/java/io/questdb/client/impl/PooledSender.java
+++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java
@@ -335,6 +335,11 @@ public void reset() {
slot.live(generation).reset();
}
+ @Override
+ public void resetSymbolDictionary() {
+ slot.live(generation).resetSymbolDictionary();
+ }
+
@Override
public Sender shortColumn(CharSequence name, short value) {
slot.live(generation).shortColumn(name, value);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java
index d9ab6026..c4bb594c 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java
@@ -45,7 +45,7 @@
/**
* The producer-side dictionary cap ({@code MAX_SYMBOL_DICTIONARY_SIZE}) as the
* application sees it: {@code symbol()} with a value that would create the
- * 1,000,001st distinct entry throws BEFORE the row is buffered, the row is
+ * 2,000,001st distinct entry throws BEFORE the row is buffered, the row is
* cancellable, and the sender keeps working with already-registered values --
* the wire never carries the refused symbol.
*/
@@ -76,7 +76,7 @@ public void testSymbolPastCapThrowsAndSenderStaysUsable() throws Exception {
sender.table("t").symbol("s", "one-too-many");
Assert.fail("expected LineSenderException past the dictionary cap");
} catch (LineSenderException expected) {
- Assert.assertTrue(expected.getMessage().contains("1000000"));
+ Assert.assertTrue(expected.getMessage().contains(String.valueOf(MAX_SYMBOL_DICTIONARY_SIZE)));
}
Assert.assertEquals("the refusal must not have grown the dictionary",
MAX_SYMBOL_DICTIONARY_SIZE, dict.size());
@@ -98,6 +98,56 @@ public void testSymbolPastCapThrowsAndSenderStaysUsable() throws Exception {
});
}
+ /**
+ * A threshold configured AT its own cap (half of the producer-side
+ * {@code MAX_SYMBOL_DICTIONARY_SIZE}), with automatic reset DISABLED,
+ * must behave exactly like the undecorated cap: the refusal still fires
+ * once the dictionary itself reaches {@code MAX_SYMBOL_DICTIONARY_SIZE},
+ * and its message still names the reset valve even though this
+ * particular sender has it switched off -- the valve is documented for
+ * senders that want it, not conditioned on this sender having chosen it.
+ *
+ * Out of scope here: whether {@code symbol_dict_reset=off} actually keeps
+ * {@code armIfEligible()} from arming. That only runs from the tail of a
+ * completed {@code flush()}, which this test never performs (the fill
+ * goes through the raw dictionary test accessor, and the one
+ * {@code Sender}-routed call throws inside {@code symbol()} before a row
+ * completes) -- an {@code isResetArmed()} assertion here would pass
+ * regardless of the knob, proving nothing. That arming-vs-flush property
+ * is pinned in {@code SymbolDictRecycleArmingTest.testArmsAtThreshold}.
+ */
+ @Test
+ public void testCapReachedWithResetDisabledStillThrowsAndNamesTheResetValve() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+ + ";symbol_dict_reset=off;symbol_dict_reset_threshold=" + (MAX_SYMBOL_DICTIONARY_SIZE / 2) + ";")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ GlobalSymbolDictionary dict = ws.getGlobalSymbolDictionaryForTest();
+ for (int i = 0; i < MAX_SYMBOL_DICTIONARY_SIZE; i++) {
+ dict.getOrAddSymbol("f" + i);
+ }
+
+ try {
+ sender.table("t").symbol("s", "one-too-many");
+ Assert.fail("expected LineSenderException past the dictionary cap");
+ } catch (LineSenderException expected) {
+ String message = expected.getMessage();
+ Assert.assertTrue("message names the limit: " + message,
+ message.contains(String.valueOf(MAX_SYMBOL_DICTIONARY_SIZE)));
+ Assert.assertTrue("message points at the reset valve: " + message,
+ message.contains("symbol_dict_reset") && message.contains("resetSymbolDictionary()"));
+ }
+ }
+ }
+ });
+ }
+
private static void waitFor(Condition condition, long timeoutMillis) throws Exception {
long deadline = System.currentTimeMillis() + timeoutMillis;
while (!condition.holds()) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
index 65d61c26..9b418a85 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
@@ -298,14 +298,14 @@ public void testSpecialCharactersInSymbols() {
@Test
public void testGetOrAddSymbol_refusesGrowthPastProtocolCap() {
- // Pre-sized so the 1M fill does not rehash its way through the test budget.
- GlobalSymbolDictionary dict = new GlobalSymbolDictionary(1 << 21);
+ // Pre-sized so the 2M fill does not rehash its way through the test budget.
+ GlobalSymbolDictionary dict = new GlobalSymbolDictionary(1 << 22);
for (int i = 0; i < QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE; i++) {
assertEquals(i, dict.getOrAddSymbol("f" + i));
}
- // Boundary: the 1,000,000th distinct symbol (id 999_999) was ACCEPTED above --
+ // Boundary: the 2,000,000th distinct symbol (id 1_999_999) was ACCEPTED above --
// the guard must refuse growth PAST the cap, not growth TO it, because the
- // server accepts a catch-up of exactly deltaStart + deltaCount == 1_000_000.
+ // server accepts a catch-up of exactly deltaStart + deltaCount == 2_000_000.
assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE, dict.size());
try {
@@ -313,9 +313,12 @@ public void testGetOrAddSymbol_refusesGrowthPastProtocolCap() {
fail("expected LineSenderException past the dictionary cap");
} catch (LineSenderException expected) {
assertTrue("message names the limit: " + expected.getMessage(),
- expected.getMessage().contains("1000000"));
+ expected.getMessage().contains("2000000"));
assertTrue("message names the recovery: " + expected.getMessage(),
expected.getMessage().contains("close this sender"));
+ assertTrue("message points at the reset valve: " + expected.getMessage(),
+ expected.getMessage().contains("symbol_dict_reset")
+ && expected.getMessage().contains("resetSymbolDictionary()"));
}
// The refusal mutated nothing: size unchanged, the refused symbol absent,
@@ -333,9 +336,9 @@ public void testGetOrAddSymbol_refusesGrowthPastProtocolCap() {
@Test
public void testProtocolCapConstantPinnedToServerValue() {
// The server-side QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE (questdb OSS) is
- // 1_000_000 and the ingress decoder rejects any delta or catch-up whose
+ // 2_000_000 and the ingress decoder rejects any delta or catch-up whose
// deltaStartId + deltaCount exceeds it. If this pin fails, the server
// constant moved and both sides must move together.
- assertEquals(1_000_000, QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE);
+ assertEquals(2_000_000, QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE);
}
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
index 5d2b9976..b3796c25 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
@@ -27,12 +27,16 @@
import io.questdb.client.Sender;
import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
import io.questdb.client.test.AbstractTest;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
import io.questdb.client.test.tools.TestUtils;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
+import java.util.concurrent.TimeUnit;
+
import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
/**
@@ -269,6 +273,128 @@ public void testCatchUpCapGapMinEscalationWindowUnsetInSnapshot() {
.get("catch_up_cap_gap_min_escalation_window_millis"));
}
+ @Test
+ public void testSymbolDictResetDefaults() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() {
+ })) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ try (Sender sender = Sender.fromConfig("ws::addr=" + LOCALHOST + ":" + port + ";")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertTrue(ws.isSymbolDictResetEnabled());
+ Assert.assertEquals(100_000, ws.getSymbolDictResetThreshold());
+ Assert.assertEquals(2_000L, ws.getSymbolDictResetMaxWaitMillis());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSymbolDictResetConfigStringRoundTrip() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() {
+ })) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ try (Sender sender = Sender.fromConfig("ws::addr=" + LOCALHOST + ":" + port
+ + ";symbol_dict_reset=off;symbol_dict_reset_threshold=500;"
+ + "symbol_dict_reset_max_wait_millis=0;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertFalse(ws.isSymbolDictResetEnabled());
+ Assert.assertEquals(500, ws.getSymbolDictResetThreshold());
+ Assert.assertEquals(0L, ws.getSymbolDictResetMaxWaitMillis());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSymbolDictResetThresholdRejectsBadValues() {
+ assertThrows("symbol_dict_reset_threshold must be > 0",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold=0;"));
+ assertThrows("symbol_dict_reset_threshold must be > 0",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold=-5;"));
+ assertThrows("symbol_dict_reset_threshold must be > 0 and <= " + (QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2),
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold="
+ + (QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2 + 1) + ";"));
+ assertThrows("symbol_dict_reset_max_wait_millis must be >= 0: -1",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_max_wait_millis=-1;"));
+ assertThrows("symbol_dict_reset_max_wait_millis is out of range",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_max_wait_millis="
+ + (Long.MAX_VALUE / 1_000_000L + 1) + ";"));
+ }
+
+ @Test
+ public void testSymbolDictResetRejectedForNonWebSocketTransport() {
+ assertThrows("symbol_dict_reset is only supported for WebSocket transport",
+ () -> Sender.builder("http::addr=" + LOCALHOST + ":9000;symbol_dict_reset=on;"));
+ assertThrows("symbol_dict_reset_threshold is only supported for WebSocket transport",
+ () -> Sender.builder("http::addr=" + LOCALHOST + ":9000;symbol_dict_reset_threshold=500;"));
+ assertThrows("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport",
+ () -> Sender.builder("http::addr=" + LOCALHOST + ":9000;symbol_dict_reset_max_wait_millis=0;"));
+ }
+
+ /**
+ * The three fluent setters carry the same transport guard as the
+ * connect-string keys, but sit on a separate code path -- pin them
+ * directly so a guard dropped from the setters alone cannot ship green.
+ */
+ @Test
+ public void testSymbolDictResetFluentSettersRejectNonWebSocketTransport() {
+ assertThrows("symbol_dict_reset is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictReset(true));
+ assertThrows("symbol_dict_reset_threshold is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictResetThreshold(500));
+ assertThrows("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictResetMaxWaitMillis(0));
+ }
+
+ /**
+ * {@code symbol_dict_reset=on} must survive the parse as {@code true} --
+ * distinct from the default-true path, which passes with the parse branch
+ * deleted. Contrast against {@code off} on an otherwise identical builder.
+ */
+ @Test
+ public void testSymbolDictResetOnParsesTrue() {
+ Assert.assertEquals(Boolean.TRUE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=on;")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset"));
+ Assert.assertEquals(Boolean.FALSE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=off;")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset"));
+ }
+
+ @Test
+ public void testSymbolDictResetRejectsInvalidValue() {
+ assertThrows("invalid symbol_dict_reset [value=banana, allowed-values=[on, off]]",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=banana;"));
+ }
+
+ /**
+ * The accepted upper edge: exactly half of {@code MAX_SYMBOL_DICTIONARY_SIZE}
+ * (1M, the re-arm floor's own cap) must pass validation -- a {@code >} ->
+ * {@code >=} regression at the bound would reject it. Both the
+ * connect-string and the fluent setter paths.
+ */
+ @Test
+ public void testSymbolDictResetThresholdAcceptsHalfCapBoundary() {
+ Assert.assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold="
+ + QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2 + ";")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset_threshold"));
+ Assert.assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2,
+ Sender.builder(Sender.Transport.WEBSOCKET)
+ .symbolDictResetThreshold(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2)
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset_threshold"));
+ }
+
@Test
public void testConnectionRefused() throws Exception {
assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
index 1ed78752..8b42b337 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
@@ -147,7 +147,7 @@ static int readVarint(byte[] buffer, int[] position) {
throw new IllegalStateException("varint truncated");
}
- static int tableCount(byte[] frame) {
+ public static int tableCount(byte[] frame) {
return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8);
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
new file mode 100644
index 00000000..faba814c
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
@@ -0,0 +1,385 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.DelegatingFilesFacade;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Covers the arming half of the symbol-dictionary recycle feature:
+ * {@code QwpWebSocketSender.armIfEligible()}, called at the tail of
+ * {@code resetTableBuffersAfterFlush()}, and the manual advisory API
+ * {@link Sender#resetSymbolDictionary()}.
+ */
+public class SymbolDictRecycleArmingTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testArmsAtThreshold() throws Exception {
+ // threshold=3, send rows with symbols a,b -> flush -> not armed;
+ // add c -> flush -> armed
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server) + "symbol_dict_reset_threshold=3;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1).atNow();
+ sender.flush();
+ Assert.assertFalse(ws.isResetArmed());
+ sender.table("t").symbol("s", "c").longColumn("v", 1).atNow();
+ sender.flush();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ /**
+ * Arming ignores {@code deltaDictEnabled} -- threshold-based
+ * evaluation must still run once the sender has degraded to full self-sufficient
+ * frames. Reaching a custom low {@code symbol_dict_reset_threshold} on a
+ * sender that also carries the fault-injecting {@code FilesFacade} needs
+ * {@code QwpWebSocketSender}'s widest {@code connect(List
+ * This {@code connect(...)} overload installs no {@link
+ * io.questdb.client.cutlass.qwp.client.QwpWebSocketSender.EngineRebuildFactory
+ * EngineRebuildFactory} (only {@code Sender.build()} does), so
+ * crossing the threshold must never actually arm -- {@code
+ * armIfEligible()} folds the capability check in ahead of the threshold
+ * comparison. That rule is instead pinned negatively here: full-dict
+ * degradation does not change that verdict either way.
+ */
+ @Test
+ public void testDoesNotArmWithoutRebuildFactory() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("arm-full-dict-sf").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir,
+ io.questdb.client.std.Files.DIR_MODE_DEFAULT));
+
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+
+ MmapFaultDictFacade ff = new MmapFaultDictFacade();
+ CursorSendEngine engine = new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff);
+ QwpWebSocketSender sender = QwpWebSocketSender.connect(
+ Collections.singletonList(new QwpWebSocketSender.Endpoint("localhost", port)),
+ null, // tlsConfig
+ 0, 0, 0L, // autoFlushRows, autoFlushBytes, autoFlushIntervalNanos
+ null, // authorizationHeader
+ false, // requestDurableAck
+ engine,
+ 5_000L, // closeFlushTimeoutMillis
+ CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS,
+ CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS,
+ CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS,
+ Sender.InitialConnectMode.OFF,
+ null, // errorHandler
+ SenderErrorDispatcher.DEFAULT_CAPACITY,
+ CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
+ QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS,
+ 0, // connectTimeoutMs
+ null, // connectionListener
+ SenderConnectionDispatcher.DEFAULT_CAPACITY,
+ CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
+ CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS,
+ CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS,
+ true, // symbolDictResetEnabled
+ 3, // symbolDictResetThresholdSymbols -- low, deliberately crossed below
+ QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS);
+ try {
+ ff.armed = true; // next dictionary mmap growth raises a recognised fault
+ sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("expected the injected mmap fault to fail this flush");
+ } catch (LineSenderException expected) {
+ // Same guard MmapFaultDegradesTest pins: the fault degrades the
+ // sender to full self-sufficient frames instead of propagating raw.
+ }
+ Assert.assertFalse("a recognised mmap access fault must degrade the sender "
+ + "to full-dict mode",
+ sender.isDeltaDictEnabledForTest());
+ Assert.assertFalse("dictionary has only 1 entry, below the threshold of 3",
+ sender.isResetArmed());
+
+ // The fault facade disarms itself after firing once, so this retry
+ // succeeds and clears pendingRowCount back to 0; "a" is now published.
+ sender.flush();
+ Assert.assertFalse("still degraded, dictionary still below threshold",
+ sender.isDeltaDictEnabledForTest());
+ Assert.assertFalse(sender.isResetArmed());
+
+ sender.table("m").symbol("s", "b").longColumn("v", 2L).atNow();
+ sender.flush();
+ Assert.assertFalse("dictionary has 2 entries, still below the threshold of 3",
+ sender.isResetArmed());
+
+ // No manual resetSymbolDictionary() call anywhere in this test: crossing
+ // the threshold, even while degraded, still must not arm -- this
+ // connect(...) overload installs no engineRebuildFactory,
+ // and that capability check now runs ahead of the threshold
+ // comparison in armIfEligible().
+ sender.table("m").symbol("s", "c").longColumn("v", 3L).atNow();
+ sender.flush();
+ Assert.assertFalse("a sender with no rebuild factory must never arm, even once "
+ + "the threshold is crossed in full-dict mode",
+ sender.isResetArmed());
+ } finally {
+ sender.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testDoesNotArmWhenDisabled() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(
+ cfg(server) + "symbol_dict_reset=off;symbol_dict_reset_threshold=2;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertFalse("symbol_dict_reset=off must never arm", ws.isResetArmed());
+ sender.table("t").symbol("s", "c").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertFalse("symbol_dict_reset=off must never arm", ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testManualResetRequestArms() throws Exception {
+ assertMemoryLeak(() -> {
+ // pendingRowCount == 0: resetSymbolDictionary() arms immediately.
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+
+ // Mid-batch: a request while a row is buffered (pendingRowCount != 0)
+ // only arms once the next flush runs armIfEligible() at its tail.
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.resetSymbolDictionary();
+ Assert.assertFalse("a mid-batch request must not arm before the next flush",
+ ws.isResetArmed());
+ sender.flush();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testReArmFloorDoublesPerSwapAndBlocksOrganicReArm() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("floor-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String config = cfg(server) + "sf_dir=" + sfDir + ";symbol_dict_reset_threshold=2;";
+ try (Sender sender = Sender.fromConfig(config)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertEquals("no swap yet: floor is 0", 0, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 0: two symbols == threshold -> arms
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ // swap #1 runs inside this table() with dictSizeAtSwap == 2
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ Assert.assertEquals("floor = 2 x size-at-swap", 4, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 1: c,d,e,f == floor -> arms again
+ sender.table("t").symbol("s", "d").longColumn("v", 2L).atNow();
+ sender.table("t").symbol("s", "e").longColumn("v", 2L).atNow();
+ sender.table("t").symbol("s", "f").longColumn("v", 2L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertTrue("size 4 >= max(threshold 2, floor 4) must arm", ws.isResetArmed());
+
+ // swap #2 with dictSizeAtSwap == 4
+ sender.table("t").symbol("s", "g").longColumn("v", 3L).atNow();
+ Assert.assertEquals(2, ws.getSymbolDictEpoch());
+ Assert.assertEquals("floor doubles again", 8, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 2: four symbols is above the threshold but below the floor
+ sender.table("t").symbol("s", "h").longColumn("v", 4L).atNow();
+ sender.table("t").symbol("s", "i").longColumn("v", 4L).atNow();
+ sender.table("t").symbol("s", "j").longColumn("v", 4L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertFalse("size 4 < floor 8 must not re-arm organically", ws.isResetArmed());
+
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("the advisory request bypasses the floor", ws.isResetArmed());
+
+ // manual swap #3 runs inside this table() with dictSizeAtSwap == 4
+ // (g,h,i,j): 2 x 4 == the current floor, so this step alone cannot
+ // tell a lowered floor from a kept one -- it only proves the manual
+ // swap runs and that the floor did not move.
+ sender.table("t").symbol("s", "k").longColumn("v", 5L).atNow();
+ Assert.assertEquals(3, ws.getSymbolDictEpoch());
+ Assert.assertEquals("2 x 4 == floor 8: unchanged either way",
+ 8, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 3 holds one symbol (k). A manual swap here sets the floor to
+ // 2 x 1 = 2 if the valve can lower it; the floor must stay at 8.
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("manual request arms below the floor", ws.isResetArmed());
+ sender.table("t").symbol("s", "l").longColumn("v", 6L).atNow();
+ Assert.assertEquals(4, ws.getSymbolDictEpoch());
+ Assert.assertEquals("a manual swap at dictionary size 1 must not lower the floor",
+ 8, ws.getResetFloorSymbolsForTesting());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testResetSymbolDictionaryOnNonWsSenderIsNoOp() throws Exception {
+ assertMemoryLeak(() -> {
+ // protocolVersion(2) skips the eager server-side settings detection
+ // connect that build() otherwise performs, so no live server is needed
+ // (see LineSenderBuilderTest.testCustomPemRootsDoNotRequirePassword).
+ try (Sender sender = Sender.builder(Sender.Transport.HTTP)
+ .address("localhost")
+ .protocolVersion(2)
+ .build()) {
+ sender.resetSymbolDictionary();
+ }
+ });
+ }
+
+ @Test
+ public void testSplitFlushPathArms() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+ // Padding inflates each table past half the cap, so the combined
+ // two-table message exceeds it while each single-table split frame fits.
+ String pad = TestUtils.repeat("x", 60);
+ try (Sender sender = Sender.fromConfig(cfg(server) + "symbol_dict_reset_threshold=2;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow();
+ sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow();
+ sender.flush();
+ Assert.assertTrue("the split-flush path shares resetTableBuffersAfterFlush's tail",
+ ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ private static TestWebSocketServer ackingServer() throws Exception {
+ TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler());
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ return server;
+ }
+
+ private static String cfg(TestWebSocketServer server) {
+ return "ws::addr=localhost:" + server.getPort() + ";";
+ }
+
+ /**
+ * ACKs every frame it receives; does not otherwise inspect the wire.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Raises a RECOGNISED mmap access fault out of the persisted dictionary's next
+ * mmap growth, once, when {@link #armed}. Copied from
+ * {@code MmapFaultDegradesTest.MmapFaultDictFacade}.
+ */
+ private static final class MmapFaultDictFacade extends DelegatingFilesFacade {
+ boolean armed;
+
+ @Override
+ public boolean isMmapAllowed() {
+ return true;
+ }
+
+ @Override
+ public long mmap(int fd, long len, long offset, int flags, int memoryTag) {
+ if (armed) {
+ armed = false;
+ throw new InternalError(
+ "a fault occurred in a recent unsafe memory access operation in compiled Java code");
+ }
+ return INSTANCE.mmap(fd, len, offset, flags, memoryTag);
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleCatchUpSkipTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleCatchUpSkipTest.java
new file mode 100644
index 00000000..bb15a55d
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleCatchUpSkipTest.java
@@ -0,0 +1,380 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.std.Compat;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Pins that the symbol-dictionary recycle's fresh connection ({@code
+ * QwpWebSocketSender.recycleForDictReset()}'s step 7 reconnect) never pays
+ * for a delta-dictionary catch-up frame, and that the state-reset it relies
+ * on to get there is not accidentally general-purpose.
+ *
+ * The catch-up mechanism itself is {@link DeltaDictCatchUpTest}'s territory
+ * ({@code CursorWebSocketSendLoop.setWireBaselineWithCatchUp}'s gate:
+ * {@code client != null && sentDictCount > 0 && hasReplayDictionaryDependency}).
+ * This suite does not re-implement or re-verify that mechanism -- it only
+ * observes the ONE fact specific to the recycle: {@code sentDictCount} on the
+ * fresh loop starts at 0 because {@code recycleForDictReset()}'s step 6
+ * rebuilds the engine on a freshly-emptied slot, whose {@code
+ * PersistedSymbolDict.recoveredSize()} is 0 -- so the loop constructor's
+ * {@code pd.recoveredSize() > 0} seed never fires, and the gate stays false
+ * for the whole first post-recycle connection. A PLAIN (non-recycle)
+ * reconnect on that same connection, by contrast, reuses the SAME loop
+ * instance whose mirror has since grown from the frames it sent -- so it DOES
+ * trip the gate. Observing both back to back in one test is the only way to
+ * prove the zero count above is the recycle's fresh-mirror property and not a
+ * blind spot in how this suite's handler counts frames.
+ *
+ * No production change is expected to make these pass. A failure here means
+ * either the fresh-mirror seeding regressed (a post-recycle connection
+ * started paying for catch-up again) or the recycle's {@code
+ * sentMaxSymbolId} reset ({@code recycleForDictReset()}'s step 6) leaked
+ * onto the ordinary reconnect path, which today never touches that
+ * baseline.
+ */
+public class SymbolDictRecycleCatchUpSkipTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * The core scenario, SF-disk mode. {@code symbol_dict_reset_threshold=3}
+ * is deliberately higher than the 2 symbols this test registers in the
+ * new epoch before forcing the unplanned drop: epoch 0 crosses the
+ * threshold on its own (a, b, x -- 3 distinct symbols), so the recycle
+ * fires exactly once, synchronously, on the "c" call. Epoch 1 then
+ * registers only c, d (2 symbols, below the threshold) before the drop,
+ * and only e (a 3rd) after it -- staying unarmed for the whole test so no
+ * SECOND recycle can sneak in and confound the "does a plain reconnect
+ * still catch up / preserve the baseline" assertions below. (A lower
+ * threshold that let epoch 1 re-arm on c, d would turn the later {@code
+ * table("e")} call into an unwanted second recycle, landing e on a 4th
+ * connection instead of a plain reconnect's 3rd -- exactly the
+ * confounder this threshold choice avoids.)
+ */
+ @Test
+ public void testRecycleSkipsCatchUpThenUnplannedReconnectBoundsCatchUpToNewEpoch() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("catchup-skip").toString();
+ SkipCatchUpHandler handler = new SkipCatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=3;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Epoch 0 (connection 1): 3 distinct symbols cross threshold=3 and arm.
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "x").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=3", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Ring drained: this table() call recycles synchronously (steps 1-6:
+ // fresh empty engine/dictionary/epoch), and "c" is then the new
+ // epoch's own first symbol. The fresh connection (2) itself is the
+ // I/O thread's job and completes asynchronously -- confirmed below,
+ // after an acked post-recycle frame proves it is up.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("epoch-1 batch must be acked before the unplanned drop",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+
+ // --- Pin 1 + 2: zero catch-up frames, dictionary tiles from 0. ---
+ // Connection 2 is the FIRST connection after the recycle: its loop's
+ // sentDictCount mirror was seeded from the fresh engine's
+ // PersistedSymbolDict.recoveredSize() == 0 (nothing survived the
+ // recycle's slot wipe), so setWireBaselineWithCatchUp's
+ // `sentDictCount > 0` gate stays false for this whole connection.
+ Assert.assertEquals("first post-recycle connection must send zero catch-up "
+ + "(zero-table) frames",
+ 0, handler.zeroTableFramesFor(2));
+ Assert.assertEquals("connection 2's dictionary must tile ids from 0 with "
+ + "exactly the new epoch's symbols, none of epoch 0's a, b, x",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+
+ // --- Positive control + pin 3: an UNPLANNED reconnect (server-side
+ // drop, no recycle involved) on this SAME connection DOES produce a
+ // catch-up frame, and that catch-up is bounded to exactly what this
+ // epoch has sent so far (c, d) -- proving both that the zero count
+ // above is a real property (not a handler blind spot) and that the
+ // recycle's fresh mirror does not somehow retain epoch 0's symbols.
+ handler.dropConnection(2);
+ waitFor(() -> handler.connectionsAccepted.get() >= 3, 5_000);
+ waitFor(() -> handler.dictFor(3).size() >= 2, 5_000);
+
+ Assert.assertTrue("an unplanned reconnect mid-epoch must still produce a "
+ + "catch-up frame",
+ handler.zeroTableFramesFor(3) >= 1);
+ Assert.assertEquals("the catch-up must bound itself to exactly this epoch's "
+ + "own symbols (c, d), never replaying epoch 0's a, b, x",
+ Arrays.asList("c", "d"), handler.dictFor(3));
+
+ // --- Pin 4: the plain reconnect preserved sentMaxSymbolId. A NEW
+ // symbol registered after it must ship with a delta start ABOVE 0.
+ // Nothing on this I/O-thread reconnect path touches sentMaxSymbolId
+ // (resetSymbolDictStateForNewConnection runs only on the foreground
+ // initial-connect path, guarded by the connected flag, and never
+ // fires here), so the producer's baseline (c, d already at ids 0, 1)
+ // survives the wire boundary and e resumes at id 2. Only
+ // recycleForDictReset()'s step 6 ever zeroes that baseline; a
+ // regression that folded the reset into a path this reconnect DOES
+ // run would re-ship the whole dictionary from deltaStart 0.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-reconnect row must still get acked",
+ sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertTrue("connection 3's post-reconnect data frame carrying the new "
+ + "symbol e must ship a delta start ABOVE the surviving "
+ + "baseline (>= 1), not 0",
+ handler.sawDeltaAboveBaselineOn(3));
+ }
+
+ Assert.assertEquals("exactly 3 connections total (epoch 0, epoch 1's first "
+ + "connection, epoch 1's unplanned reconnect)",
+ 3, handler.connectionsAccepted.get());
+ }
+ });
+ }
+
+ /**
+ * Pin 5: the recycle's step 7 reconnect funnels through {@code
+ * ensureConnected()}'s {@code ASYNC} arm exactly like any other initial
+ * connect, which ends up at the same {@code swapClient} catch-up gate as
+ * the SYNC-mode scenario above. Mirrors {@code
+ * SymbolDictRecycleMemoryModeTest#testRecycleUnderAsyncInitialConnect},
+ * but in SF-disk mode (this suite's mode throughout) rather than memory
+ * mode, and asserts the zero-catch-up property instead of just the
+ * delta-start/dictionary-content pair that test already covers.
+ */
+ @Test
+ public void testRecycleUnderAsyncInitialConnectSendsZeroCatchUpFrames() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("catchup-skip-async").toString();
+ SkipCatchUpHandler handler = new SkipCatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";initial_connect_retry=async;symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Let the I/O thread complete the deferred initial connect before
+ // driving any traffic through it (see
+ // SymbolDictRecycleMemoryModeTest.awaitWasEverConnected).
+ awaitWasEverConnected(ws);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Recycles synchronously on the producer thread for steps 1-6; step
+ // 7's reconnect just re-arms the ASYNC path -- the actual handshake
+ // happens on the I/O thread and must be awaited via the ack below.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm immediately (producer-side state)",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked once the async "
+ + "I/O thread completes the fresh handshake",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, handler.connectionsAccepted.get());
+ }
+
+ Assert.assertEquals("exactly 2 connections total", 2, handler.connectionsAccepted.get());
+ Assert.assertEquals("the ASYNC path funnels through the same swapClient catch-up "
+ + "gate -- the first post-recycle connection must still send zero "
+ + "catch-up frames",
+ 0, handler.zeroTableFramesFor(2));
+ Assert.assertEquals("connection 2's dictionary must hold only the post-recycle "
+ + "symbols, not a, b",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+ }
+ });
+ }
+
+ /**
+ * Spins until the I/O thread has completed the deferred ASYNC initial
+ * connect (mirrors {@code SymbolDictRecycleMemoryModeTest}'s helper of
+ * the same name).
+ */
+ private static void awaitWasEverConnected(QwpWebSocketSender ws) {
+ long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (!ws.wasEverConnected()) {
+ if (System.nanoTime() > deadlineNanos) {
+ throw new AssertionError("I/O thread did not complete the async initial "
+ + "connect within 5s");
+ }
+ Compat.onSpinWait();
+ }
+ }
+
+ private static void waitFor(BoolCondition cond, long timeoutMillis) {
+ long deadline = System.currentTimeMillis() + timeoutMillis;
+ while (System.currentTimeMillis() < deadline) {
+ if (cond.test()) return;
+ try {
+ Thread.sleep(20);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ Assert.fail("interrupted");
+ }
+ }
+ Assert.fail("waitFor timed out");
+ }
+
+ @FunctionalInterface
+ private interface BoolCondition {
+ boolean test();
+ }
+
+ /**
+ * Reconstructs each connection's per-connection delta dictionary (mirrors
+ * {@code DeltaDictCatchUpTest.CatchUpHandler} / {@code
+ * SymbolDictRecycleTest.RecycleHandler}), counts zero-table (catch-up)
+ * frames per connection, tracks whether any data frame on a connection
+ * carried a delta start above 0, and -- unlike the sibling handlers --
+ * exposes {@link #dropConnection(int)} so the TEST THREAD can force an
+ * unplanned drop asynchronously, independent of the ack-driven close a
+ * handler normally does from inside {@code onBinaryMessage}.
+ */
+ private static class SkipCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ private final List
+ * The wedge: {@code SegmentManager}'s trim-sync hook runs unconditionally once
+ * per service pass, so parking the worker there leaves it un-joinable exactly
+ * the way a stalled disk/NFS syscall does, and a shrunken
+ * {@code workerJoinTimeoutMillis} lets the close's bounded join give up
+ * promptly (the same recipe as {@code SlotLockReleasedContractTest}).
+ */
+public class SymbolDictRecycleDeferredCloseTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * A transient wedge: the worker un-wedges while the recycle is parked in
+ * its deferred-close await. The recycle must ride the stall out and
+ * complete -- fresh epoch, rebuilt engine, sender fully usable -- instead
+ * of latching terminal on the retained flock.
+ */
+ @Test(timeout = 60_000L)
+ public void testRecycleSurvivesDeferredEngineClose() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-deferred-survive").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean wedgeFired = new AtomicBoolean();
+ AtomicReference
+ * Every test that rolls the base does so on a sender BEFORE its first connect
+ * (via {@link #createRolledSender}), never on an already-connected one:
+ * {@code rollFsnEpochBase}'s precondition forbids rolling while a live
+ * {@code CursorWebSocketSendLoop} is attached (its {@code externalFsnBase} is a
+ * construction-time snapshot, never updated on a live loop -- see that method's
+ * javadoc). Tests that need a realistic pre-roll FSN to roll by first drive a
+ * SEPARATE, ordinarily-connected sender against the same server to publish and
+ * ack a batch, close it, then hand that FSN to {@code createRolledSender} for a
+ * second, fresh sender/engine -- modelling the post-recycle engine that
+ * restarts its raw FSNs at 0.
+ */
+public class SymbolDictRecycleFsnContinuityTest {
+
+ /**
+ * Rolls a FRESH sender/engine (never published-to raw watermark starts at -1), not
+ * the already-connected one that produced {@code fsn1}: {@code rollFsnEpochBase}'s
+ * precondition forbids rolling while a live loop is attached (see its javadoc), and
+ * -- independent of that -- an already-connected sender's engine keeps its raw
+ * watermark across the roll, which would make this test pass even with the
+ * translation deleted (raw {@code ackedFsn() == fsn1 >= fsn1} regardless of any
+ * epoch math). Only a genuinely fresh engine (raw {@code ackedFsn() == -1}) makes
+ * the pre-roll short-circuit the ONLY way {@code awaitAckedFsn(fsn1, 0)} can return
+ * true here.
+ */
+ @Test
+ public void testPreRollTargetAnswersTrueAfterRoll() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ long fsn1;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue("setup: the batch must actually be acked before the roll",
+ sender1.drain(5_000));
+ }
+
+ QwpWebSocketSender sender2 = createRolledSender(server, fsn1);
+ try {
+ long t0 = System.nanoTime();
+ Assert.assertTrue("a target FSN from a pre-recycle epoch must be reported acked "
+ + "immediately -- it was proven acked before the swap",
+ sender2.awaitAckedFsn(fsn1, 0));
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+ Assert.assertTrue("must short-circuit, not poll: took " + elapsedMs + "ms",
+ elapsedMs < 200);
+ } finally {
+ sender2.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testPostRollSequencesExceedAllPreRoll() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ long fsn1;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue(sender1.drain(5_000));
+ }
+
+ QwpWebSocketSender sender2 = createRolledSender(server, fsn1);
+ try {
+ long newBase = sender2.getFsnEpochBaseForTesting();
+ Assert.assertEquals(fsn1 + 1, newBase);
+
+ sender2.table("t").longColumn("v", 2L).atNow();
+ long fsn2 = sender2.flushAndGetSequence();
+ Assert.assertTrue(sender2.drain(5_000));
+
+ Assert.assertTrue("post-roll FSN must exceed every pre-roll FSN: fsn2=" + fsn2
+ + " fsn1=" + fsn1,
+ fsn2 > fsn1);
+ // sender2's engine is genuinely fresh (raw publishedFsn() starts at -1), so
+ // its first-ever flush publishes raw 0. The exact-equality check is strictly
+ // stronger than ">" alone: it also catches an off-by-one in the roll formula
+ // (e.g. fsnEpochBase += lastPublishedFsn instead of + 1L), which the ">"
+ // check above would not.
+ Assert.assertEquals(newBase, fsn2);
+ } finally {
+ sender2.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testGetAckedFsnMonotoneAcrossRoll() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ long w;
+ long lastPublishedFsn;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ long fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue(sender1.drain(5_000));
+ w = sender1.getAckedFsn();
+ lastPublishedFsn = fsn1;
+ Assert.assertEquals("sanity: single-batch acked watermark must match its own FSN",
+ fsn1, w);
+ }
+
+ // A fresh sender/engine models the post-recycle engine that restarts its
+ // internal FSNs at 0; rolling its epoch base by the outgoing epoch's last
+ // published FSN is exactly what the recycle swap does in production.
+ QwpWebSocketSender sender2 = createRolledSender(server, lastPublishedFsn);
+ try {
+ long newBase = sender2.getFsnEpochBaseForTesting();
+ Assert.assertEquals(lastPublishedFsn + 1, newBase);
+
+ Assert.assertEquals("before any new ack, getAckedFsn must read the synthetic "
+ + "watermark: one past the last external FSN the outgoing epoch "
+ + "ever reported",
+ newBase - 1, sender2.getAckedFsn());
+ Assert.assertTrue(sender2.getAckedFsn() >= w);
+
+ sender2.table("t").longColumn("v", 2L).atNow();
+ sender2.flush();
+ Assert.assertTrue(sender2.drain(5_000));
+ Assert.assertTrue("a new ack must advance the watermark past the synthetic "
+ + "post-roll value",
+ sender2.getAckedFsn() > newBase - 1);
+ } finally {
+ sender2.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * The raw-feed bug test: without the {@code drain()} fix, a rolled epoch base makes
+ * the raw {@code cursorEngine.publishedFsn()} target look like it belongs to a
+ * pre-recycle epoch (its raw value is smaller than the rolled base), so the fixed
+ * {@code awaitAckedFsn} would short-circuit {@code true} on an un-rebased target --
+ * even though the frame was never actually acked. Must fail (spurious true) before
+ * {@code drain()} translates its target by {@code fsnEpochBase}.
+ */
+ @Test
+ public void testDrainAfterRollWaitsForNewFrames() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // Roll well past the raw FSNs this fresh engine will ever publish, so a
+ // missing translation in drain() would make its raw target look pre-roll.
+ // Must roll before the sender's first connect (see rollFsnEpochBase's
+ // precondition: cursorSendLoop must be null).
+ QwpWebSocketSender sender = createRolledSender(server, 999L);
+ try {
+ sender.table("foo").longColumn("v", 1L).atNow();
+ boolean drainedEarly = sender.drain(200);
+ Assert.assertFalse("drain() must not spuriously report the new frame acked just "
+ + "because its raw FSN is smaller than the rolled epoch base",
+ drainedEarly);
+
+ handler.releaseAcks();
+ Assert.assertTrue("drain() must return true once the real ack arrives",
+ sender.drain(5_000));
+ } finally {
+ handler.releaseAcks();
+ sender.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * {@link SenderError#getFromFsn()} / {@link SenderError#getToFsn()} surface synchronously
+ * via {@link LineSenderServerException#getServerError()}, unreachable by any
+ * dispatcher-side rebase -- the loop must rebase the span itself. Rolls the epoch base
+ * BEFORE the sender's first connect (the loop's {@code externalFsnBase} is frozen at
+ * construction) so the terminal NACK's span is built under a nonzero base.
+ */
+ @Test
+ public void testSenderErrorSpansCarryExternalFsns() throws Exception {
+ assertMemoryLeak(() -> {
+ TerminalNackHandler handler = new TerminalNackHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ AtomicReference
+ * Also covers the two permanent recycle-metrics getters ({@code getSymbolDictEpoch()},
+ * {@code getSymbolDictResetStarvationTimeouts()}).
+ */
+public class SymbolDictRecycleHealingTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testMetricsAfterTwoRecycles() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("metrics-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(0, ws.getSymbolDictResetStarvationTimeouts());
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ long framesSentBeforeRecycle = ws.getTotalFramesSent();
+ long acksBeforeRecycle = ws.getTotalAcks();
+ Assert.assertTrue("setup: the first flush must have been sent and acked, got sent="
+ + framesSentBeforeRecycle + " acks=" + acksBeforeRecycle,
+ framesSentBeforeRecycle >= 1 && acksBeforeRecycle >= 1);
+ Assert.assertTrue("armed: 2 distinct symbols crossed threshold=2", ws.isResetArmed());
+
+ // Ring drained -> this table() call recycles synchronously: epoch 1.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ Assert.assertEquals("no starvation wait was deliberately triggered",
+ 0, ws.getSymbolDictResetStarvationTimeouts());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ // Sender-lifetime, not per-loop: the rebuilt loop must keep counting
+ // where the outgoing one stopped, so a monitor differencing these
+ // never sees a negative delta across a recycle.
+ Assert.assertTrue("frames sent must carry across the recycle: before="
+ + framesSentBeforeRecycle + " after=" + ws.getTotalFramesSent(),
+ ws.getTotalFramesSent() > framesSentBeforeRecycle);
+ Assert.assertTrue("acks must carry across the recycle: before="
+ + acksBeforeRecycle + " after=" + ws.getTotalAcks(),
+ ws.getTotalAcks() > acksBeforeRecycle);
+ long framesSentAfterFirstRecycle = ws.getTotalFramesSent();
+ long acksAfterFirstRecycle = ws.getTotalAcks();
+ // The anti-thrash floor (resetFloorSymbols = 2x the first swap's
+ // dictSizeAtSwap = 4) keeps c,d (2 symbols, == threshold but < floor)
+ // from re-arming on their own; a manual request bypasses the floor by
+ // design, so drive the second recycle through resetSymbolDictionary().
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("manual reset request bypasses the re-arm floor",
+ ws.isResetArmed());
+
+ // Ring drained again -> second recycle: epoch 2.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ Assert.assertEquals(2, ws.getSymbolDictEpoch());
+ Assert.assertEquals("still no starvation wait was deliberately triggered",
+ 0, ws.getSymbolDictResetStarvationTimeouts());
+
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertTrue("frames sent must carry across the second recycle too: after first="
+ + framesSentAfterFirstRecycle + " now=" + ws.getTotalFramesSent(),
+ ws.getTotalFramesSent() > framesSentAfterFirstRecycle);
+ Assert.assertTrue("acks must carry across the second recycle too: after first="
+ + acksAfterFirstRecycle + " now=" + ws.getTotalAcks(),
+ ws.getTotalAcks() > acksAfterFirstRecycle);
+ // A recycle's own reconnect runs on the I/O loop's ASYNC path, whose
+ // first attempt counts as a reconnect attempt and, on success, a
+ // reconnect -- two recycles, so at least two of each.
+ Assert.assertTrue("attempts=" + ws.getTotalReconnectAttempts(),
+ ws.getTotalReconnectAttempts() >= 2);
+ Assert.assertTrue("reconnects=" + ws.getTotalReconnectsSucceeded(),
+ ws.getTotalReconnectsSucceeded() >= 2);
+ }
+ }
+ });
+ }
+
+ /**
+ * The recovery-side sibling of {@code MmapFaultDegradesTest.testMmapAccessFaultDegradesPersistInsteadOfPropagating}:
+ * once the sender has degraded to full self-sufficient frames, the underlying fault clears,
+ * and a recycle rebuilds the engine, the fresh engine must re-derive delta-dict mode from
+ * scratch rather than staying degraded forever. Wire evidence: the first post-recycle frame
+ * (a fresh, empty dictionary) starts a delta at 0; the SECOND post-recycle frame, which
+ * introduces exactly one more symbol, starts its delta where the first one left off and
+ * carries only that one new entry -- the shape only delta mode produces. In full-dict mode
+ * every frame re-ships the whole dictionary from id 0 (see
+ * {@code QwpWebSocketSender.symbolDeltaBaseline()}: confirmedMaxId is permanently -1), so
+ * this pair of frames could not look like this if healing had not taken effect.
+ */
+ @Test
+ public void testRecycleHealsFullDictDegradeBackToDeltaMode() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-sf").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir,
+ io.questdb.client.std.Files.DIR_MODE_DEFAULT));
+
+ CapturingAckHandler handler = new CapturingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+
+ HealableMmapFaultFacade ff = new HealableMmapFaultFacade();
+ CursorSendEngine engine = new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff);
+ QwpWebSocketSender sender = buildSender(port, engine, 100_000);
+ // connect() never installs an engineRebuildFactory (only Sender.build() does),
+ // so the recycle would otherwise be a no-op. Install one that rebuilds on the
+ // SAME slot with the SAME (healable) facade -- mirroring the real factory
+ // Sender.build() installs, minus the FilesFacade seam Sender.fromConfig lacks.
+ sender.setEngineRebuildFactory(() -> new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff));
+ try {
+ Assert.assertTrue("must start in delta mode", sender.isDeltaDictEnabledForTest());
+
+ // Degrade mid-life: fault the dictionary's next mmap growth.
+ ff.armed = true;
+ sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("expected the injected mmap fault to fail this flush");
+ } catch (LineSenderException expected) {
+ // same guard MmapFaultDegradesTest pins
+ Assert.assertTrue("the fault must be reported as a sender error, not a "
+ + "raw InternalError: " + expected.getMessage(),
+ expected.getMessage().contains(
+ "failed to persist symbol dictionary before publish"));
+ }
+ Assert.assertFalse("a recognised mmap access fault must degrade the sender",
+ sender.isDeltaDictEnabledForTest());
+
+ // Heal the facade. The retry below does not itself touch mmap --
+ // persistNewSymbolsBeforePublish short-circuits once !deltaDictEnabled --
+ // so healing here matters only for what the fresh post-recycle engine sees.
+ ff.armed = false;
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("the degraded retry must still ingest the row",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+
+ // Drained: arm and trigger the recycle.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(sender.isResetArmed());
+ sender.table("m").symbol("s", "b").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", sender.isResetArmed());
+ Assert.assertEquals(1, sender.getSymbolDictEpoch());
+
+ // The rebuilt engine re-derives delta-dict mode from scratch (a fresh,
+ // empty dictionary always opens cleanly at construction -- see this
+ // test's persistent-fault sibling for why this alone does not prove the
+ // facade was healed). The discriminating check is below, after the first
+ // post-recycle append.
+ Assert.assertTrue(sender.isDeltaDictEnabledForTest());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+
+ // Discriminating check: fsn2's flush was the fresh engine's first
+ // append. A still-armed facade would have degraded it there (as the
+ // persistent-fault sibling proves against the identical setup) -- staying
+ // true here is real evidence the heal took effect, not just an artifact
+ // of fresh-engine construction never touching mmap.
+ Assert.assertTrue("a healed facade must let the fresh engine's first "
+ + "post-recycle append succeed and keep delta mode enabled",
+ sender.isDeltaDictEnabledForTest());
+
+ sender.table("m").symbol("s", "c").longColumn("v", 3L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+
+ List
+ * WARNING -- recycle-only handler, do not copy into a plain-reconnect test.
+ * The per-connection sequence reset below assumes every connection change
+ * is a recycle, i.e. that a fresh engine is behind the new connection and
+ * its raw FSNs really do restart at 0. On an ordinary reconnect the SAME
+ * engine survives and keeps counting, so resetting here would ack frames
+ * the sender never published and silently advance its watermark past
+ * unsent data.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ // A rebuilt engine restarts its raw FSNs at 0 (externalFsnBase absorbs the
+ // offset), and the ack sequence below is applied as a raw engine FSN -- so
+ // acking a recycle's fresh connection against the outgoing connection's
+ // sequence would ack frames that were never published. Reset per connection,
+ // matching CapturingAckHandler below.
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /** ACKs every frame and records the raw bytes of every data frame, grouped by connection. */
+ private static class CapturingAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final List
+ * The recycle swap's eight steps ({@code QwpWebSocketSender.recycleForDictReset()})
+ * were written against the store-and-forward slot lifecycle, but the factory's
+ * {@code slotPath == null} arm, {@code CursorSendEngine}'s file-less close, and the
+ * barrier itself are all mode-agnostic by construction -- nothing in
+ * {@code maybeRecycleForDictReset()} or the swap checks whether the sender is
+ * SF-backed. This suite pins that: every scenario {@code SymbolDictRecycleTest}
+ * proves for a disk-backed sender must hold identically for a {@code Sender.fromConfig}
+ * sender built with no {@code sf_dir} at all. No production change is expected to
+ * make these pass; a failure here means the swap accidentally gated something
+ * on store-and-forward being present.
+ */
+public class SymbolDictRecycleMemoryModeTest {
+
+ @Test
+ public void testRecycleAtEmptyBacklog() throws Exception {
+ assertMemoryLeak(() -> {
+ RecycleHandler handler = new RecycleHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ // No sf_dir: memory mode. Everything else mirrors
+ // SymbolDictRecycleTest#testRecycleAtEmptyBacklog exactly.
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Ring drained, no row in progress: this table() call must
+ // recycle synchronously, exactly as in SF mode. The fresh
+ // WebSocket handshake is the I/O thread's job and completes
+ // asynchronously -- it is asserted below, after an acked
+ // post-recycle frame proves the connection is up.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']',
+ fsn2 > fsn1);
+ }
+
+ Assert.assertEquals("exactly 2 connections total", 2, handler.connectionsAccepted.get());
+ Assert.assertEquals("connection 2's first data frame must carry deltaStart == 0 "
+ + "(a fresh, empty dictionary)",
+ 0, handler.conn2FirstFrameDeltaStart);
+ Assert.assertEquals("connection 2's dictionary must hold only the post-recycle "
+ + "symbols, not a, b",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+ }
+ });
+ }
+
+ /**
+ * Strengthens {@link #testRecycleAtEmptyBacklog} into a content oracle: every
+ * row before and after the recycle carries a distinct symbol value, and this
+ * asserts the server observed the full, gap-free, duplicate-free
+ * symbol-registration sequence across both connections; every row carries a
+ * distinct symbol, so that sequence mirrors the rows frame for frame -- not
+ * just a spot check of the boundary frame. Proves the epoch swap loses (and
+ * doesn't duplicate) nothing that was ever acked, in memory mode exactly as
+ * {@code testPostRecycleSlotContents} proves the persisted-dictionary shape
+ * in SF mode.
+ */
+ @Test
+ public void testRecycleLosesNothingAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ RecycleHandler handler = new RecycleHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=2;";
+
+ List
+ * (a) proves the recycle's step 2 ({@code cursorSendLoop.close()}) correctly
+ * joins an I/O thread that is itself mid-reconnect (not idle, not yet given
+ * up), and that step 7 no longer recovers the connection on the calling
+ * thread -- it defers to the I/O loop, so the swap returns promptly and the
+ * producer never observes the outage -- exercising
+ * {@code CursorWebSocketSendLoop.close()}'s "handles both states" contract
+ * under a real outage rather than a synthetic one.
+ *
+ * (b) proves the swap only ever tears down the producer's OWN cursor
+ * engine/I/O loop: an orphan drainer's engine and loop are entirely separate
+ * objects owned by {@code BackgroundDrainerPool}, so a recycle firing while a
+ * drain is in flight must leave the drain untouched and able to complete
+ * afterward.
+ */
+public class SymbolDictRecycleOutageTest {
+
+ private static final String ORPHAN_MARKER_SYMBOL = "orphan-marker-1";
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * Kills the server out from under an armed, fully-drained sender, waits
+ * for the pre-recycle I/O thread to actually enter its own reconnect
+ * loop (not just assumed via a fixed sleep) -- so the recycle's step 2
+ * ({@code cursorSendLoop.close()}) provably joins a MID-reconnect
+ * thread -- then triggers the recycle inline, on the calling thread.
+ * {@code reconnect_max_duration_millis} bounds only the sender's initial
+ * connect; under the store-and-forward contract step 7 no longer
+ * re-enters {@code connectWithRetry} on the producer thread, so the
+ * triggering {@code table()} call must return well within that budget
+ * even though the endpoint is still down when it fires. The main thread
+ * revives a fresh server on the same port after asserting the bound,
+ * mirroring {@code ReconnectTest}'s down-then-up realism.
+ */
+ @Test
+ public void testSyncModeRecycleDoesNotBlockProducerDuringOutage() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("outage-recycle").toString();
+ AckAllHandler firstHandler = new AckAllHandler();
+ int port;
+ try (TestWebSocketServer server = new TestWebSocketServer(firstHandler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2"
+ + ";reconnect_initial_backoff_millis=20"
+ + ";reconnect_max_backoff_millis=80"
+ + ";reconnect_max_duration_millis=6000;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the outage",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Kill the connection AND the listener -- a real outage, not
+ // just a dropped socket the same server would re-accept
+ // instantly.
+ server.close();
+
+ // Confirm the pre-recycle I/O thread actually entered its
+ // own reconnect loop against the now-refused port before we
+ // trigger the recycle -- so step 2's close() below is
+ // provably joining a MID-reconnect thread, not one that
+ // simply hasn't noticed the drop yet.
+ long attemptDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (ws.getTotalReconnectAttempts() == 0 && System.nanoTime() < attemptDeadline) {
+ Thread.sleep(5);
+ }
+ Assert.assertTrue("pre-recycle I/O thread must have entered reconnect before "
+ + "the triggering table() call",
+ ws.getTotalReconnectAttempts() > 0);
+
+ // The recycle must return promptly: reconnect_max_duration_millis
+ // governs only the initial connect, and step 7 defers to the
+ // I/O loop instead of re-entering connectWithRetry on the
+ // producer thread.
+ long startNanos = System.nanoTime();
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long elapsedMillis = (System.nanoTime() - startNanos) / 1_000_000L;
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals("recycle must complete despite the outage",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertTrue("the swap must not block the producer on the reconnect "
+ + "budget [elapsedMillis=" + elapsedMillis + ']',
+ elapsedMillis < 3_000);
+
+ long fsn2 = sender.flushAndGetSequence();
+ OutageRecycleHandler revivedHandler = new OutageRecycleHandler();
+ try (TestWebSocketServer revived =
+ new TestWebSocketServer(revivedHandler, false, null, port)) {
+ revived.start();
+ Assert.assertTrue(revived.awaitStart(5, TimeUnit.SECONDS));
+ Assert.assertTrue("the outage-window row must land once reconnected",
+ sender.awaitAckedFsn(fsn2, 10_000));
+ Assert.assertTrue(fsn2 > fsn1);
+ Assert.assertEquals(0, revivedHandler.firstFrameDeltaStart);
+ Assert.assertEquals(Collections.singletonList("c"), revivedHandler.dict());
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * Default configuration: no {@code reconnect_*} knob and no
+ * {@code initial_connect_retry}, so the builder resolves
+ * {@code initialConnectMode} to OFF. Under the store-and-forward
+ * contract, step 7 no longer opens a connection on the calling thread
+ * at all -- it defers to the I/O loop, so the triggering {@code table()}
+ * call must return normally even while the endpoint refuses
+ * connections.
+ *
+ * Proves the swap commits exactly one epoch and disarms without the
+ * caller ever observing a transport failure, that the flush right after
+ * publishes into the fresh epoch's SF slot, and that once the endpoint
+ * returns on the same port the I/O loop's own reconnect replays every
+ * row sent during the outage with zero loss -- reconnecting only, never
+ * re-running a teardown step and never swapping a second time.
+ */
+ @Test
+ public void testDefaultConfigRecycleBuffersThroughOutage() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("default-config-outage").toString();
+ AckAllHandler firstHandler = new AckAllHandler();
+ int port;
+ try (TestWebSocketServer server = new TestWebSocketServer(firstHandler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertTrue("the recycle must be on under a default configuration",
+ ws.isSymbolDictResetEnabled());
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the outage",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Kill the listener AND the live connection. The ring is
+ // drained, so the sender-level connected flag is still true
+ // and the next table() call fires the recycle into a wire
+ // that is already down.
+ server.close();
+
+ // The ring is drained, so the next table() fires the recycle
+ // into a wire that is already down. The swap must complete AND
+ // return normally -- the reconnect is the I/O loop's job, so
+ // no transport failure may reach the producer. "c" registers
+ // into the fresh dictionary after the swap's
+ // resetSymbolDictStateForNewConnection but before the wire is
+ // up, which keeps pinning the drained-guard: a deferred
+ // connect that cleared the batch watermark would ship a row
+ // pointing at an id the server never received.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertTrue("wasEverConnected() must stay sticky across the recycle's "
+ + "rebuilt loop while the endpoint is still down -- the fresh "
+ + "loop must not report 'never connected' just because it is a "
+ + "new loop instance",
+ ws.wasEverConnected());
+ Assert.assertEquals("the swap must commit exactly one epoch",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertFalse("a committed swap disarms", ws.isResetArmed());
+
+ // Producer keeps working against the dead endpoint: the
+ // flush publishes into the fresh epoch's SF slot.
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN",
+ fsn2 > fsn1);
+
+ // Endpoint back on the SAME port: the I/O loop's own
+ // reconnect must land the buffered rows -- zero loss.
+ OutageRecycleHandler revivedHandler = new OutageRecycleHandler();
+ try (TestWebSocketServer revived =
+ new TestWebSocketServer(revivedHandler, false, null, port)) {
+ revived.start();
+ Assert.assertTrue(revived.awaitStart(5, TimeUnit.SECONDS));
+
+ Assert.assertTrue("rows sent during the outage must replay once "
+ + "the endpoint returns",
+ sender.awaitAckedFsn(fsn2, 10_000));
+ Assert.assertEquals("the recovery reconnects only -- no second swap",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertEquals("the fresh connection's first frame must carry a "
+ + "fresh (empty) dictionary, not a, b",
+ 0, revivedHandler.firstFrameDeltaStart);
+ Assert.assertEquals(Collections.singletonList("c"), revivedHandler.dict());
+
+ // And the epoch keeps extending normally from there.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertEquals("later batches must extend the same fresh dictionary",
+ Arrays.asList("c", "e"), revivedHandler.dict());
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * An orphan drainer's engine and I/O loop are objects entirely separate
+ * from the foreground sender's own {@code cursorEngine}/{@code
+ * cursorSendLoop} -- {@code BackgroundDrainerPool} owns them. Seeds a
+ * sibling orphan slot (mirrors {@code OrphanScanIntegrationTest}'s ghost
+ * recipe), lets the drainer adopt it and get its replay frame gated on
+ * the wire, then arms and fires a recycle on the foreground stream while
+ * the drain is provably still in flight. The recycle must leave the
+ * drain untouched: releasing the gate afterward still lets it complete,
+ * and every one of the three streams (pre-recycle foreground,
+ * post-recycle foreground, drained orphan) lands with the right symbol.
+ */
+ @Test
+ public void testOrphanDrainerSurvivesRecycleMidDrain() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("outage-orphan-drain").toString();
+
+ // Phase 1: seed a sibling orphan slot. The ghost writes one row
+ // carrying a uniquely-marked symbol and dies without ever being
+ // acked -- same recipe as OrphanScanIntegrationTest.
+ SilentHandler ghostSilent = new SilentHandler();
+ try (TestWebSocketServer ghostServer = new TestWebSocketServer(ghostSilent)) {
+ ghostServer.start();
+ Assert.assertTrue(ghostServer.awaitStart(5, TimeUnit.SECONDS));
+ String ghostCfg = "ws::addr=localhost:" + ghostServer.getPort()
+ + ";sf_dir=" + sfDir + ";sender_id=ghost;close_flush_timeout_millis=0;";
+ try (Sender ghost = Sender.fromConfig(ghostCfg)) {
+ ghost.table("orphaned").symbol("s", ORPHAN_MARKER_SYMBOL).longColumn("v", 99L).atNow();
+ ghost.flush();
+ Assert.assertTrue("ghost frame must reach the wire before close",
+ ghostSilent.awaitFrame(5, TimeUnit.SECONDS));
+ }
+ }
+ Assert.assertEquals("ghost slot must be a candidate orphan",
+ 1, OrphanScanner.scan(sfDir, "primary").size());
+
+ // Phase 2: one server serves both the primary sender and the
+ // orphan drainer it spawns. Gating is CONTENT-based (whichever
+ // connection ships the ghost's marker symbol), not
+ // connection-order-based -- the drainer's connect can race the
+ // primary's own first flush, and content-based gating stays
+ // correct regardless of which one wins that race.
+ PrimaryAndOrphanHandler handler = new PrimaryAndOrphanHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String primaryCfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";sender_id=primary;drain_orphans=on;symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(primaryCfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Let the drainer discover + adopt the ghost slot and get
+ // its replay frame gated on the wire before touching the
+ // foreground stream at all -- proves the two run
+ // concurrently, not sequentially.
+ Assert.assertTrue("orphan drainer must ship its replay frame",
+ handler.awaitOrphanFrame(10, TimeUnit.SECONDS));
+
+ // Arm + fire the recycle on the foreground stream. These
+ // frames carry none of the orphan marker, so they get
+ // acked immediately regardless of the drain's state.
+ sender.table("t").symbol("s", "pre-a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "pre-b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Recycle fires synchronously here, tearing down + rebuilding
+ // ONLY the foreground's own cursor engine/I/O loop.
+ sender.table("t").symbol("s", "post-c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle row must land on the fresh connection",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > fsn1);
+
+ // The drain must still be exactly where it was -- gated,
+ // not failed, not restarted -- proving the recycle never
+ // reached into the drainer's separate stack.
+ Assert.assertFalse("the drainer's connection must not have been touched by "
+ + "the foreground's recycle", handler.orphanAcked());
+
+ // Now release the drainer's gate: a drain that survived the
+ // recycle untouched must still be able to complete.
+ handler.releaseOrphan();
+ long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (OrphanScanner.scan(sfDir, "primary").size() > 0
+ && System.nanoTime() < deadlineNanos) {
+ Thread.sleep(10);
+ }
+ Assert.assertEquals("orphan drainer must complete the drain after the recycle",
+ 0, OrphanScanner.scan(sfDir, "primary").size());
+ }
+
+ // Per-row symbol correctness for all three streams.
+ Assert.assertEquals("pre-recycle foreground stream",
+ Arrays.asList("pre-a", "pre-b"), handler.dictContaining("pre-a"));
+ Assert.assertEquals("post-recycle foreground stream",
+ Collections.singletonList("post-c"), handler.dictContaining("post-c"));
+ Assert.assertEquals("drained orphan stream",
+ Collections.singletonList(ORPHAN_MARKER_SYMBOL),
+ handler.dictContaining(ORPHAN_MARKER_SYMBOL));
+ }
+ });
+ }
+
+ /**
+ * Invariant B's seed: a 401 handshake rejection AFTER a recycle is
+ * transient only because the swap seeds the fresh loop with
+ * markEverConnected() -- without it, the fresh loop would classify the
+ * same 401 as a pre-first-connect endpoint-policy failure and latch a
+ * terminal, turning a transient auth blip into data loss.
+ */
+ @Test(timeout = 60_000L)
+ public void testPostRecycleEndpointPolicyRejectionIsTransient() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("post-recycle-401").toString();
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: arm batch must be acked", sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ // Every handshake from here on is met with 401 -- including
+ // the fresh post-recycle loop's very first connect.
+ server.setRejectWithStatus(401, "Unauthorized");
+
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow(); // recycle fires here
+ Assert.assertEquals("the swap itself needs no connection", 1, ws.getSymbolDictEpoch());
+ Assert.assertTrue("the seed must survive the swap", ws.wasEverConnected());
+
+ // Producing keeps working: the rejection is transient under
+ // Invariant B, so rows buffer and nothing latches.
+ long fsn2 = sender.flushAndGetSequence();
+
+ // The fresh loop's connect is deferred to its own I/O thread
+ // and races this thread, so wait for the server to actually
+ // observe (and reject) at least one handshake before checking
+ // anything below -- otherwise this thread could relent before
+ // the fresh loop's first attempt ever reaches the wire, and
+ // the assertions that follow would pass without exercising
+ // the 401 path at all.
+ long rejectDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (server.statusRejectCount() == 0 && System.nanoTime() < rejectDeadline) {
+ Thread.sleep(2);
+ }
+ Assert.assertTrue("the fresh post-recycle loop must actually hit the 401 "
+ + "before this test can exercise Invariant B's seed",
+ server.statusRejectCount() > 0);
+ Assert.assertNull("must not latch a terminal on a post-recycle 401",
+ ws.getLastTerminalError());
+
+ // Clear BEFORE close() -- drainOnClose would otherwise burn its
+ // whole flush budget against the rejecting server.
+ server.setRejectWithStatus(0, null);
+ Assert.assertTrue("buffered rows must land once the endpoint relents",
+ sender.awaitAckedFsn(fsn2, 10_000));
+ Assert.assertTrue(ws.wasEverConnected());
+ }
+ }
+ });
+ }
+
+ /** ACKs every frame it receives immediately; does not otherwise inspect the wire. */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Reconstructs the single connection it expects (the recycle's
+ * post-outage reconnect) and records the delta-start id of its first
+ * data frame. Tracks by connection identity like
+ * {@code SymbolDictRecycleTest.RecycleHandler} so a partially-established
+ * retry that never sends data cannot corrupt the state of the connection
+ * that actually does.
+ */
+ private static class OutageRecycleHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final List
+ * {@link SymbolDictRecycleTest} and {@link SymbolDictRecycleMemoryModeTest}
+ * pin the swap itself; {@link SymbolDictRecycleArmingTest} pins how
+ * {@code resetArmed} flips true; {@link SymbolDictRecycleStarvationTest} pins
+ * the bounded blocking wait for an unacked backlog. This suite pins the
+ * OPPOSITE: every condition under which an armed sender must keep working
+ * normally and NOT recycle, until that condition clears -- at which point
+ * the still-armed request fires as a positive control in the same test.
+ * Every test asserts both halves: no recycle (connection count and
+ * {@code getSymbolDictEpoch()} unchanged) AND that ingestion keeps
+ * working (a row lands and gets acked) both before and after the eventual
+ * recycle.
+ */
+public class SymbolDictRecycleRefusalTest {
+
+ /**
+ * The most basic refusal: a batch that itself crossed the arming
+ * threshold is still unacked when the very next {@code table()} call
+ * checks the barrier. {@code symbol_dict_reset_max_wait_millis=0}
+ * disables the (separately-pinned, {@link SymbolDictRecycleStarvationTest})
+ * blocking wait, so every refusal here is instant and this test stays
+ * purely about the ring-drained guard. Repeated {@code table()} calls
+ * spread over a real span of wall-clock time (not one instantaneous
+ * check) prove the recycle does not fire late, either -- the whole
+ * mechanism is synchronous and producer-thread-driven, but a bounded
+ * settle window is the only way a test can actually witness that rather
+ * than assume it.
+ */
+ @Test
+ public void testUnackedBacklogRefusesUntilAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0;";
+
+ // ackGate closes before sender, so an early failure still lets close() drain
+ try (Sender sender = Sender.fromConfig(cfg); AutoCloseable ackGate = handler::releaseAcks) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ awaitHandshakes(server, 1);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence(); // ack withheld by the handler
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, server.handshakeCount());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("recycle must not fire while the arming batch is unacked",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: release the acks and prove the still-armed
+ // recycle fires on the very next drained table() call.
+ handler.releaseAcks();
+ Assert.assertTrue("setup: the arming batch must get acked once released",
+ sender.awaitAckedFsn(fsn1, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the backlog drains", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ // Ingestion continues on the fresh epoch.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * Isolates the {@code pendingRowCount != 0} guard from the ring-drained
+ * guard {@link #testUnackedBacklogRefusesUntilAcked} pins: the arming
+ * batch's ack is released and awaited WHILE a third row sits buffered
+ * (committed via {@code atNow()}, but never flushed -- {@code auto_flush_rows}
+ * is set well above 1 so it does not auto-flush). By the time the
+ * settle-window loop runs, the ring itself is fully drained, so any
+ * refusal it observes can only be this guard, not the earlier one.
+ */
+ @Test
+ public void testPendingRowCountRefuses() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0"
+ + ";auto_flush_rows=10;";
+
+ try (Sender sender = Sender.fromConfig(cfg); AutoCloseable ackGate = handler::releaseAcks) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ awaitHandshakes(server, 1);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence(); // ack withheld by the handler
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ // Ring not drained yet: refused by the OTHER guard, which just
+ // lets execution fall through so a new row can be buffered.
+ sender.table("t");
+ Assert.assertTrue(ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // A third row, committed but never flushed: pendingRowCount=1,
+ // far under auto_flush_rows=10, so it stays buffered.
+ sender.symbol("s", "c").longColumn("v", 2L).atNow();
+
+ // Drain the arming batch -- from here on the ring itself is
+ // fully drained, isolating the pendingRowCount guard.
+ handler.releaseAcks();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("recycle must not fire while a row is buffered "
+ + "unflushed, even with the ring otherwise drained",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: flush the buffered row, then the
+ // still-armed recycle fires on the next table() call.
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the buffered batch is flushed "
+ + "and acked",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn3 > fsn2);
+ }
+ }
+ });
+ }
+
+ /**
+ * A row under construction (columns set, {@code atNow()} not yet called)
+ * refuses the barrier two different ways depending on the next
+ * {@code table()} call's table name. The same-name case is the sharper
+ * proof: {@code table()}'s resetArmed check runs BEFORE the
+ * same-table-name fast path that would otherwise skip straight past
+ * everything, so this is the only way to prove the hook actually sits
+ * ahead of that shortcut. The different-name case falls through to the
+ * pre-existing "cannot switch tables while row is in progress" guard
+ * instead -- a thrown exception, not a recycle, and not a new failure
+ * mode this feature introduced.
+ */
+ @Test
+ public void testInProgressRowRefuses() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ awaitHandshakes(server, 1);
+
+ // Start a row but do not commit it: symbol() registers "a"
+ // into the dictionary immediately, yet the row itself stays
+ // in progress until atNow() runs.
+ sender.table("t").symbol("s", "a");
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Arm WHILE the row is in progress: pendingRowCount is still
+ // 0 (an in-progress row is not counted as pending), so the
+ // manual request arms immediately even though a row is
+ // genuinely mid-flight.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+
+ for (int i = 0; i < 3; i++) {
+ sender.table("t"); // same name -- fast path would skip past everything
+ Assert.assertTrue("recycle must not fire while a row is in progress",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(20);
+ }
+
+ LineSenderException thrown = null;
+ try {
+ sender.table("other");
+ Assert.fail("expected 'cannot switch tables' while a row is in progress");
+ } catch (LineSenderException e) {
+ thrown = e;
+ }
+ Assert.assertNotNull(thrown);
+ Assert.assertTrue("unexpected message: " + thrown.getMessage(),
+ thrown.getMessage().contains("cannot switch tables while row is in progress"));
+ Assert.assertTrue("the failed table-switch attempt must not have consumed "
+ + "the arming",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ // Complete the row: ingestion still works after both refusals.
+ sender.longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("nothing yet consumed the arming", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Positive control: with the row complete and the batch
+ // acked, the still-armed recycle fires on the next call.
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the row completes and the ring "
+ + "drains",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * The one data-safety-critical refusal, mirroring
+ * {@code SymbolDictRecycleStarvationTest#testDeferredCommitGroupSkipsWait}
+ * but for the barrier itself rather than the blocking-wait futility
+ * guard: the server withholds acks for {@code FLAG_DEFER_COMMIT} frames
+ * by design until the closing commit lands, so {@code isRingDrained()}
+ * stays false for as long as the group is open, however long that is.
+ * {@code symbol_dict_reset_max_wait_millis=0} keeps this test orthogonal
+ * to the (separately-pinned) starvation-wait timing.
+ */
+ @Test
+ public void testDeferredCommitGroupRefusesUntilCommitAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ DeferAwareAckHandler handler = new DeferAwareAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ awaitHandshakes(server, 1);
+ ws.setDeferCommit(true);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush(); // deferred frame -- server withholds its ack by design
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("an open deferred-commit group must never let the "
+ + "recycle fire -- the server withholds its ack until "
+ + "the closing commit",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: close the group, wait for its
+ // (retroactive) ack, and prove the still-armed recycle
+ // fires next.
+ ws.setDeferCommit(false);
+ long commitFsn = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the commit must get acked",
+ sender.awaitAckedFsn(commitFsn, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the group is committed and acked",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > commitFsn);
+ }
+ }
+ });
+ }
+
+ /**
+ * A manual {@code resetSymbolDictionary()} call arms {@code resetArmed}
+ * regardless of connection state ({@code armIfEligible()} touches only
+ * producer-side fields), so it can go through before the sender has ever
+ * connected -- modelled the same way
+ * {@code SymbolDictRecycleFsnContinuityTest} builds unconnected senders:
+ * {@link QwpWebSocketSender#createForTesting} plus a manually attached
+ * engine, with {@link QwpWebSocketSender#setEngineRebuildFactory} filled
+ * in (unlike {@code createForTesting}'s production counterparts, a
+ * connect()-built sender normally has none -- see
+ * {@code SymbolDictRecycleTest#testConnectBuiltSenderNeverRecyclesWithoutFactory})
+ * so the deferred request can actually execute once connected. The very
+ * next {@code table()} call -- still pre-connect -- must defer rather
+ * than NPE: {@code !connected} refuses the barrier before it ever
+ * touches the cursor engine or I/O loop.
+ */
+ @Test
+ public void testManualResetBeforeFirstConnectDeferred() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", port);
+ try {
+ CursorSendEngine engine = new CursorSendEngine(
+ null, 4L * 1024 * 1024, 128L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS);
+ sender.setCursorEngine(engine, true);
+ sender.setEngineRebuildFactory(() -> new CursorSendEngine(
+ null, 4L * 1024 * 1024, 128L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS));
+
+ // Manual request before the sender has ever connected.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("a manual request arms immediately, independent of "
+ + "connection state",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+ Assert.assertEquals(0, server.handshakeCount());
+
+ // table()'s barrier check runs here while still pre-connect
+ // (ensureConnected() only runs later, inside atNow()'s
+ // sendRow()) -- must defer quietly, not NPE.
+ sender.table("t").longColumn("v", 1L).atNow();
+ awaitHandshakes(server, 1);
+ Assert.assertTrue("still armed -- deferred, not consumed",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("flush alone does not consume the arming -- only table() "
+ + "does",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+
+ // Positive control: now connected and drained, the
+ // deferred request executes on the next table() call.
+ sender.table("t");
+ Assert.assertFalse("the deferred request must execute once connected and "
+ + "drained",
+ sender.isResetArmed());
+ Assert.assertEquals(1, sender.getSymbolDictEpoch());
+
+ sender.table("t").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ } finally {
+ sender.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * {@code reset()} discards a buffered-but-never-shipped row -- including
+ * reclaiming any symbol id it registered but never sent, via the same
+ * {@code truncateTo} mechanism the {@code BatchTooLargeForCapException}
+ * remedy documents. This proves that discard is compatible with an
+ * already-armed swap: after {@code reset()} clears the in-progress row
+ * that was the ONLY thing refusing the barrier, every guard is
+ * satisfied (connected, no pending row, no in-progress row, ring
+ * drained from the earlier shipped batch), so the next {@code table()}
+ * call recycles -- observed here to fire deterministically, not
+ * probabilistically, once those guards clear. It is compatible with
+ * {@code reset()}'s own reclaim: the swap replaces the whole dictionary
+ * object outright (step 6 of {@code recycleForDictReset()}), so
+ * whatever {@code truncateTo} did to the outgoing instance is moot --
+ * the swap subsumes it.
+ */
+ @Test
+ public void testResetDiscardsBufferedRowThenArmedSwapFires() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=3;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // A real shipped batch: two distinct symbols, below the
+ // threshold of 3, so nothing arms yet.
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertFalse("dictionary has only 2 entries, below the threshold of 3",
+ ws.isResetArmed());
+
+ // Start (but never commit) a third row -- registers "c",
+ // crossing the threshold, but arming is only ever
+ // evaluated at a flush's tail or by resetSymbolDictionary(),
+ // neither of which has run yet.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L);
+ Assert.assertFalse(ws.isResetArmed());
+
+ // Arm explicitly while the row is still in progress --
+ // the in-progress-row guard refuses table(), exactly as
+ // testInProgressRowRefuses proves.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ sender.table("t"); // refused: row "c" is in progress
+ Assert.assertTrue(ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ // Discard the buffered row -- reset() drops the
+ // in-progress row AND reclaims "c"'s never-shipped id.
+ sender.reset();
+
+ // Every barrier guard is now satisfied: connected, no
+ // pending row (reset cleared it), no in-progress row
+ // (reset cleared it), ring drained (a, b were acked
+ // before any of this). The armed swap fires here.
+ sender.table("t");
+ Assert.assertFalse("the armed swap fires once reset() clears the blocking "
+ + "in-progress row",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ // Ingestion continues correctly post-swap: a fresh row
+ // lands and gets acked with no exception.
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * Polls (5s deadline) until the server has completed at least
+ * {@code expected} handshakes, then asserts exactly that many. The server
+ * counts a handshake on its own thread AFTER writing the 101, while the
+ * client returns from the upgrade as soon as it has READ it, so a bare
+ * {@code handshakeCount()} check right after a connect, or after a flush
+ * whose ack is deliberately withheld, can observe the count one step
+ * behind. Once an ack has been awaited the count is settled: the ack is
+ * sent from that same server thread, after the increment.
+ */
+ private static void awaitHandshakes(TestWebSocketServer server, int expected) throws InterruptedException {
+ long deadline = System.currentTimeMillis() + 5_000;
+ while (System.currentTimeMillis() < deadline && server.handshakeCount() < expected) {
+ Thread.sleep(10);
+ }
+ Assert.assertEquals(expected, server.handshakeCount());
+ }
+
+ /**
+ * ACKs every frame it receives; does not otherwise inspect the wire.
+ * Resets its wire sequence per new connection, mirroring
+ * {@code SymbolDictRecycleTest.RecycleHandler}, so post-recycle
+ * ingestion on the fresh connection acks correctly too.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Acks every non-deferred frame immediately, but withholds acks for any
+ * frame carrying {@code FLAG_DEFER_COMMIT} -- the real server's ack
+ * contract for an open deferred-commit group. Mirrors
+ * {@code SymbolDictRecycleStarvationTest.DeferAwareAckHandler}, plus a
+ * per-connection wire-sequence reset so ingestion on the post-recycle
+ * connection acks correctly too.
+ */
+ private static class DeferAwareAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ long seq = nextSeq.getAndIncrement();
+ boolean deferred = data.length > 5 && (data[5] & FLAG_DEFER_COMMIT) != 0;
+ if (deferred) {
+ return; // withhold the ack -- the group is still open
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(seq));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Receives frames but withholds every ack until {@link #releaseAcks()}
+ * is called, so a refusal-guard test provably has an unacknowledged
+ * target to refuse on. Mirrors
+ * {@code SymbolDictRecycleFsnContinuityTest.GatedAckHandler} /
+ * {@code SymbolDictRecycleStarvationTest.GatedAckHandler}, plus a
+ * per-connection wire-sequence reset so ingestion on the post-recycle
+ * connection acks correctly too.
+ */
+ private static class GatedAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final CountDownLatch released = new CountDownLatch(1);
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ if (!released.await(20, TimeUnit.SECONDS)) {
+ throw new AssertionError("refusal-guard witness never released the ack gate");
+ }
+ synchronized (this) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ }
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+
+ void releaseAcks() {
+ released.countDown();
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
new file mode 100644
index 00000000..6b89e274
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
@@ -0,0 +1,514 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * The two verdicts {@code QwpWebSocketSender.completeRecycleRebuild} reaches
+ * when the recycle's step-4 rebuild comes back
+ * {@code wasRecoveredFromDisk()} -- i.e. when the outgoing engine's
+ * fully-drained close did NOT leave the slot empty.
+ *
+ * The acked-leftover recipe injects the unlink failure the way
+ * {@code CursorSendEngineCloseUnlinkFailureTest} does: it drops write
+ * permission on the slot directory (POSIX unlink needs a writable parent), so
+ * that test skips on Windows and wherever permissions are not enforced (root).
+ */
+public class SymbolDictRecycleSlotHealTest {
+
+ /**
+ * Big enough for the real QWP frames the sender appends to the rebuilt
+ * engine after the heal, and identical in the prep helpers so recovery
+ * reads the doctored segments back at the size they were written with.
+ */
+ private static final long SEGMENT_BYTES = 1024L * 1024L;
+ private static final int PAYLOAD_BYTES = 32;
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * A benign fully-drained close verdict (segment unlink
+ * transiently failed; watermark retained by design) must not brick the
+ * recycle. The rebuild recovers fully-acked leftovers; the sender heals by
+ * closing the recovered engine (which retries the unlink) and rebuilding
+ * once more.
+ */
+ @Test(timeout = 60_000L)
+ public void testRecoveredFullyAckedLeftoversHealAndRecycleCompletes() throws Exception {
+ assertMemoryLeak(() -> {
+ // Phase 1: doctor a slot -- fully-acked frames whose close-time
+ // unlink failed (CursorSendEngineCloseUnlinkFailureTest's recipe).
+ String doctoredSlot = temporaryFolder.getRoot().toPath()
+ .resolve("doctored-slot").toString();
+ prepareFullyAckedLeftoverSlot(doctoredSlot);
+
+ // Phase 2: a live sender whose rebuild factory lands on that slot.
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(config(server, sfDir))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+
+ AtomicInteger rebuilds = new AtomicInteger();
+ ws.setEngineRebuildFactory(() -> {
+ rebuilds.incrementAndGet();
+ return new CursorSendEngine(doctoredSlot, SEGMENT_BYTES);
+ });
+
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ // Recycle: rebuild #1 recovers the acked leftovers -> heal
+ // -> rebuild #2 stands on a genuinely empty slot.
+ sender.table("t").symbol("s", "b").longColumn("v", 2L).atNow();
+
+ Assert.assertEquals("heal must close the recovered engine and rebuild again",
+ 2, rebuilds.get());
+ Assert.assertEquals("the recycle must have committed",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ // The heal's close retried the unlink the outgoing close
+ // could not do, so the engine the swap committed on stands
+ // on a genuinely emptied slot -- not on the leftovers.
+ Assert.assertFalse("the recycle must commit on a non-recovered engine",
+ ws.getCursorEngineForTesting().wasRecoveredFromDisk());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-heal batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']', fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * The heal closes the engine that recovered the leftovers. When that engine's
+ * SF worker is wedged, its close returns with the slot flock retained, exactly
+ * like the outgoing engine's close in step 3 -- and must be awaited the same
+ * way, or rebuild #2 collides with the retained flock.
+ */
+ @Test(timeout = 60_000L)
+ public void testHealRidesOutADeferredCloseOfTheRecoveredEngine() throws Exception {
+ assertMemoryLeak(() -> {
+ String doctoredSlot = temporaryFolder.getRoot().toPath().resolve("doctored-deferred").toString();
+ prepareFullyAckedLeftoverSlot(doctoredSlot);
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-deferred-sf").toString();
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicReference
+ * This test does NOT claim the resume heals a remainder wider than the
+ * cap: the re-registration is one-shot and delta mode ships no chunks on
+ * the ordinary flush path, so such an epoch stays wedged on the split
+ * pre-flight -- a narrower population than before the clamp, which
+ * re-shipped all of [0..sentMaxSymbolId] on the next flush.
+ */
+ @Test(timeout = 60_000L)
+ public void testResumePartialPublishClampsWatermarkToCoverage() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("resume-partial").toString();
+ ChunkCaptureHandler handler = new ChunkCaptureHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(2048);
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";";
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertTrue("precondition: SF slot must give delta mode",
+ ws.isDeltaDictEnabledForTest());
+ // One long symbol per flush: a single frame carrying all three
+ // (~4.5 KB) would trip the split pre-flight at cap 2048.
+ String[] longSymbols = {longSymbol('a'), longSymbol('b'), longSymbol('c')};
+ for (int i = 0; i < longSymbols.length; i++) {
+ sender.table("t").symbol("s", longSymbols[i]).longColumn("v", i).atNow();
+ sender.flush();
+ Assert.assertTrue("setup: flush " + i + " must drain", sender.drain(5_000));
+ }
+ Assert.assertEquals("setup: baseline covers the three long symbols",
+ 2, ws.getSentMaxSymbolIdForTesting());
+
+ ws.forceCloseLoopAbandonForTesting();
+ AtomicInteger chunkCalls = new AtomicInteger();
+ ws.setChunkPublishFaultForTesting(() -> {
+ if (chunkCalls.incrementAndGet() == 3) {
+ throw new RuntimeException("injected mid-publish chunk fault");
+ }
+ });
+ try {
+ // The resume degrades inside this call; it must NOT throw.
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ } finally {
+ ws.setChunkPublishFaultForTesting(null);
+ }
+ Assert.assertEquals("exactly three chunk publishes must have been attempted",
+ 3, chunkCalls.get());
+ Assert.assertEquals("watermark must equal the ringed coverage (chunks [0..0], [1..1])",
+ 1, ws.getSentMaxSymbolIdForTesting());
+ Assert.assertFalse("the orphaned chunks' deferred group must be closed",
+ ws.hasDeferredMessagesForTesting());
+
+ long fsn = sender.flushAndGetSequence();
+ Assert.assertTrue("the post-abandon batch must land on the fresh loop",
+ sender.awaitAckedFsn(fsn, 10_000));
+ Assert.assertEquals("chunks [0..1] replayed, then the data frame re-ships id 2 and adds id 3",
+ Arrays.asList(longSymbols[0], longSymbols[1], longSymbols[2], "d"), handler.dict());
+ Assert.assertEquals("first data frame's delta must start at coverage + 1",
+ 2, handler.firstDataFrameDeltaStart);
+ }
+ }
+ });
+ }
+
+ /**
+ * The memory-mode half of the coverage invariant {@link
+ * #testResumePartialPublishClampsWatermarkToCoverage()}'s javadoc argues but
+ * cannot exercise on an SF slot: with no persisted dictionary,
+ * {@code QwpWebSocketSender.reclaimUnsentSymbolIds}' floor IS the watermark
+ * itself (no {@code pd.size()} to raise it), so a watermark left BELOW the
+ * ringed coverage would let {@code reset()} reclaim an id a chunk already on
+ * the ring defines, and a later row could rebind it to a different string.
+ * Drives {@code reset()} after the same faulted resume and asserts only the
+ * two ringed symbols survive -- the assertion the SF twin cannot make, since
+ * the persisted dictionary's size pins its reclaim floor at 3 whatever the
+ * watermark reads.
+ */
+ @Test(timeout = 60_000L)
+ public void testResumePartialPublishInMemoryModeKeepsReclaimFloorAtCoverage() throws Exception {
+ assertMemoryLeak(() -> {
+ ChunkCaptureHandler handler = new ChunkCaptureHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(2048);
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";";
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertTrue("memory mode is always delta", ws.isDeltaDictEnabledForTest());
+ String[] longSymbols = {longSymbol('a'), longSymbol('b'), longSymbol('c')};
+ for (int i = 0; i < longSymbols.length; i++) {
+ sender.table("t").symbol("s", longSymbols[i]).longColumn("v", i).atNow();
+ sender.flush();
+ Assert.assertTrue("setup: flush " + i + " must drain", sender.drain(5_000));
+ }
+ Assert.assertEquals("setup: baseline covers the three long symbols",
+ 2, ws.getSentMaxSymbolIdForTesting());
+
+ ws.forceCloseLoopAbandonForTesting();
+ AtomicInteger chunkCalls = new AtomicInteger();
+ ws.setChunkPublishFaultForTesting(() -> {
+ if (chunkCalls.incrementAndGet() == 3) {
+ throw new RuntimeException("injected mid-publish chunk fault");
+ }
+ });
+ try {
+ // The resume degrades inside this call; it must NOT throw. The
+ // staged "d" row itself is never flushed -- reset() below discards it.
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ } finally {
+ ws.setChunkPublishFaultForTesting(null);
+ }
+ Assert.assertEquals("exactly three chunk publishes must have been attempted",
+ 3, chunkCalls.get());
+ Assert.assertEquals("watermark must equal the ringed coverage (chunks [0..0], [1..1])",
+ 1, ws.getSentMaxSymbolIdForTesting());
+ Assert.assertFalse("the orphaned chunks' deferred group must be closed",
+ ws.hasDeferredMessagesForTesting());
+
+ // The reclaim-floor half: abandon the staged row and reclaim
+ // everything above the watermark. With no persisted dictionary to
+ // raise the floor, only the two symbols the ring actually holds
+ // (longA, longB) may survive.
+ sender.reset();
+ Assert.assertEquals("reclaim floor must sit at coverage + 1: ids at or below "
+ + "the watermark are on the ring",
+ 2, ws.getGlobalSymbolDictionaryForTest().size());
+
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ long fsn = sender.flushAndGetSequence();
+ Assert.assertTrue("the post-reset batch must land on the fresh loop",
+ sender.awaitAckedFsn(fsn, 10_000));
+ Assert.assertEquals("chunks [0..1] replayed, then the data frame defines id 2 = e",
+ Arrays.asList(longSymbols[0], longSymbols[1], "e"), handler.dict());
+ Assert.assertEquals("the reclaimed id must restart the delta at coverage + 1",
+ 2, handler.firstDataFrameDeltaStart);
+ }
+ }
+ });
+ }
+
+ private static String longSymbol(char c) {
+ char[] chars = new char[1500];
+ Arrays.fill(chars, c);
+ return new String(chars);
+ }
+
+ /**
+ * A live symbol set larger than the threshold must not thrash the
+ * recycle: after a swap, re-arming requires the dictionary to reach
+ * max(threshold, 2 * size-at-swap).
+ */
+ @Test
+ public void testLiveSetAboveThresholdDoesNotThrash() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ // threshold=4; the live set has 6 distinct symbols
+ try (Sender sender = Sender.fromConfig(cfg(server) + "symbol_dict_reset_threshold=4;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ String[] live = {"s0", "s1", "s2", "s3", "s4", "s5"};
+ sendLiveSet(sender, live); // registers 6 distinct -> arms
+ sender.table("t"); // barrier -> recycle #1
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ // Refill from the SAME live pool three times over: 6 is above
+ // the threshold but below the doubled floor (12) -> no re-arm.
+ for (int pass = 0; pass < 3; pass++) {
+ sendLiveSet(sender, live);
+ sender.table("t");
+ }
+ Assert.assertEquals("a bounded live set must not re-trigger the recycle",
+ 1, ws.getSymbolDictEpoch());
+ // Genuine growth past the floor DOES re-arm: 12 fresh symbols.
+ String[] grown = new String[12];
+ for (int i = 0; i < 12; i++) {
+ grown[i] = "g" + i;
+ }
+ sendLiveSet(sender, grown);
+ sender.table("t");
+ Assert.assertEquals(2, ws.getSymbolDictEpoch());
+ }
+ }
+ });
+ }
+
+ private void sendLiveSet(Sender sender, String[] symbols) throws Exception {
+ for (String s : symbols) {
+ sender.table("t").symbol("s", s).longColumn("v", 1L).atNow();
+ }
+ long f = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(f, 5_000));
+ }
+
+ private static void awaitKind(List
+ * It must be positive: a continuous producer keeps frames in flight at every
+ * row start and never exposes a drained instant on its own, so a zero default
+ * would make the recycle unreachable for exactly the population that crosses
+ * the threshold.
+ *
+ * It must stay well below the sender pool's default acquire timeout: a pooled
+ * sender inherits an armed recycle at give-back, and the next borrower's first
+ * {@code table()} call may pay the whole wait while holding its lease. Other
+ * threads blocked in {@code acquire} must not time out before that borrower
+ * releases. The pool default is read back through the builder's resolved
+ * snapshot (a parse-only build with {@code sender_pool_min=0} connects
+ * nothing), the same way {@code PoolConfigHonoredTest} does.
+ */
+public class SymbolDictResetDefaultsTest {
+
+ @Test
+ public void testDefaultWaitIsPositive() {
+ Assert.assertTrue("the default starvation wait must be positive, or a continuous "
+ + "producer never recycles",
+ QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS > 0);
+ }
+
+ @Test
+ public void testDefaultWaitStaysBelowPoolAcquireTimeout() {
+ QuestDBBuilder b = QuestDB.builder().fromConfig("ws::addr=127.0.0.1:1;"
+ + "sender_pool_min=0;sender_pool_max=1;query_pool_min=0;query_pool_max=1;");
+ b.build().close();
+ long acquireTimeoutMillis = (Long) b.poolConfigSnapshotForTest().get("acquire_timeout_ms");
+ long waitMillis = QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS;
+ Assert.assertTrue("default starvation wait " + waitMillis + "ms must stay at or under half "
+ + "the default pool acquire timeout " + acquireTimeoutMillis + "ms, or a "
+ + "borrower paying the wait while holding its lease can starve other acquirers",
+ waitMillis * 2 <= acquireTimeoutMillis);
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java
index fc1b9257..da3c79e9 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java
@@ -76,7 +76,8 @@ public void closeOwnershipSnapshotNeverClaimsAnUnsurfacedError() {
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
0,
0,
- CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
+ CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN,
+ 0L);
loop.start();
// Race close()'s exact ownership snapshot against the latch
// transition, stopping once the latch has landed. Nothing in
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendCountersTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendCountersTest.java
new file mode 100644
index 00000000..8f0003b6
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendCountersTest.java
@@ -0,0 +1,72 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendCounters;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class CursorSendCountersTest {
+
+ @Test
+ public void testAddAllFoldsEveryFieldWithNonZeroValuesOnBothSides() {
+ CursorSendCounters a = new CursorSendCounters();
+ a.acks.set(1);
+ a.backpressureStalls.set(2);
+ a.framesReplayed.set(3);
+ a.framesSent.set(4);
+ a.reconnectAttempts.set(5);
+ a.reconnects.set(6);
+ a.serverErrors.set(7);
+
+ CursorSendCounters b = new CursorSendCounters();
+ b.acks.set(10);
+ b.backpressureStalls.set(20);
+ b.framesReplayed.set(30);
+ b.framesSent.set(40);
+ b.reconnectAttempts.set(50);
+ b.reconnects.set(60);
+ b.serverErrors.set(70);
+
+ a.addAll(b);
+
+ assertEquals("acks must sum both sides", 11, a.acks.get());
+ assertEquals("backpressureStalls must sum both sides", 22, a.backpressureStalls.get());
+ assertEquals("framesReplayed must sum both sides", 33, a.framesReplayed.get());
+ assertEquals("framesSent must sum both sides", 44, a.framesSent.get());
+ assertEquals("reconnectAttempts must sum both sides", 55, a.reconnectAttempts.get());
+ assertEquals("reconnects must sum both sides", 66, a.reconnects.get());
+ assertEquals("serverErrors must sum both sides", 77, a.serverErrors.get());
+
+ assertEquals("addAll must not mutate the argument's acks", 10, b.acks.get());
+ assertEquals("addAll must not mutate the argument's backpressureStalls", 20, b.backpressureStalls.get());
+ assertEquals("addAll must not mutate the argument's framesReplayed", 30, b.framesReplayed.get());
+ assertEquals("addAll must not mutate the argument's framesSent", 40, b.framesSent.get());
+ assertEquals("addAll must not mutate the argument's reconnectAttempts", 50, b.reconnectAttempts.get());
+ assertEquals("addAll must not mutate the argument's reconnects", 60, b.reconnects.get());
+ assertEquals("addAll must not mutate the argument's serverErrors", 70, b.serverErrors.get());
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
index 998b6e58..f083f60a 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
@@ -26,6 +26,7 @@
import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendCounters;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
@@ -421,6 +422,33 @@ public void testAppendBlockingThrowsOnDeadlineExpiryUnderCap() throws Exception
});
}
+ @Test
+ public void testAdoptCountersSharesTheInstanceAndFoldsOnlyItsOwnCounter() throws Exception {
+ try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) {
+ // Seed the engine's DEFAULT holder with a foreign total on a counter the
+ // engine never writes, so the fold's scope -- backpressureStalls only,
+ // not the whole holder -- is observable.
+ engine.getCountersForTesting().framesSent.set(99);
+ engine.getCountersForTesting().backpressureStalls.set(2);
+
+ CursorSendCounters shared = new CursorSendCounters();
+ shared.backpressureStalls.set(5);
+ engine.adoptCounters(shared);
+ assertEquals("only the engine-owned counter is folded", 7, shared.backpressureStalls.get());
+ assertEquals("framesSent is the loop's counter, not the engine's -- must not be folded",
+ 0, shared.framesSent.get());
+ assertEquals("getter must read the adopted instance", 7, engine.getTotalBackpressureStalls());
+
+ // Idempotent: re-adopting the same instance must not fold again.
+ engine.adoptCounters(shared);
+ assertEquals("a second adoptCounters on the same holder must be a no-op",
+ 7, shared.backpressureStalls.get());
+
+ shared.backpressureStalls.incrementAndGet();
+ assertEquals("getter must read the shared instance, not a copy", 8, engine.getTotalBackpressureStalls());
+ }
+ }
+
@Test
public void testAppendOrFsnReturnsBackpressureWhenSpareUnavailable() throws Exception {
TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
index ec828aae..2bae99e5 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
@@ -1177,7 +1177,7 @@ private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReje
CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
0L, TimeUnit.HOURS.toMillis(1),
- CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
+ CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN, 0L);
loopRef[0] = loop;
try {
seedMirror(loop, TestUtils.repeat("x", 200));
@@ -1331,7 +1331,7 @@ private CursorWebSocketSendLoop newLoop(
CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
0L, capGapWindowMillis,
- CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
+ CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN, 0L);
}
private CursorWebSocketSendLoop newForegroundLoop(
@@ -1437,7 +1437,7 @@ private void assertConnectLoopEntry(boolean reenterWithCapGap) throws Exception
CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
0L, 0L,
- CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
+ CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN, 0L);
loopRef[0] = loop;
try {
seedMirror(loop, TestUtils.repeat("x", 200));
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java
index 902df846..5aff5e5f 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java
@@ -100,7 +100,8 @@ public void testFirstConnectCatchUpFailureKeepsStartupTerminalArmed() throws Exc
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
0L,
0L,
- CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
+ CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND,
+ 0L);
try {
seedMirror(loop, "sym0"); // non-empty mirror => swapClient runs the catch-up
appendFrame(engine, (byte) 1);
@@ -170,7 +171,8 @@ private void assertAsyncInitialForegroundSurfacesTerminal(
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
0L,
0L,
- CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
+ CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND,
+ 0L);
try {
appendFrame(engine, (byte) 1);
loop.start();
@@ -227,7 +229,8 @@ private void assertForegroundRecovers(boolean durableAck, FailureSupplier failur
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
0L,
0L,
- CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
+ CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND,
+ 0L);
// Wire an error sink. Retrying is what the store-and-forward contract
// demands, but until dispatchRetriedEndpointPolicyFailure existed the retry
// was programmatically INVISIBLE: dispatchError ran only in the terminal
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java
index 130f60b4..bbc92f3e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java
@@ -25,6 +25,7 @@
package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendCounters;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
import io.questdb.client.std.Unsafe;
import org.junit.Assert;
@@ -35,7 +36,6 @@
import java.lang.reflect.Method;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
/**
* Regression coverage (M3): {@code catch (Throwable)} in the reconnect
@@ -160,8 +160,7 @@ private static void wireReconnectPlumbing(CursorWebSocketSendLoop loop,
};
setField(loop, "reconnectFactory", factory);
setField(loop, "running", true);
- setField(loop, "totalReconnectAttempts", new AtomicLong());
- setField(loop, "totalReconnects", new AtomicLong());
+ setField(loop, "counters", new CursorSendCounters());
}
private static CursorWebSocketSendLoop newBareLoop() throws Exception {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
index b215e476..5de5ada4 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
@@ -341,7 +341,7 @@ private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine)
},
0, 1,
false, 0L, 3, 0L, 0L,
- CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
+ CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN, 0L);
}
private static CursorWebSocketSendLoop newForegroundLoop(CursorSendEngine engine) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java
index 79870469..f242bba7 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java
@@ -25,6 +25,7 @@
package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendCounters;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
import org.junit.Assert;
@@ -136,6 +137,40 @@ public void zeroBackoffConnectFailuresKeepRetryingInsteadOfKillingIoThread() thr
}
}
+ /**
+ * The sender hands every loop generation its own sender-lifetime counters
+ * before start(); a swap under a running I/O thread could lose increments,
+ * so adoption after start() is refused.
+ */
+ @Test(timeout = 30_000)
+ public void adoptCountersSharesBeforeStartAndRefusesAfter() throws Exception {
+ try (CursorSendEngine engine = new CursorSendEngine(
+ sfDir.getRoot().getAbsolutePath(), 16_384)) {
+ CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+ null, engine, 0, 1_000_000L,
+ () -> {
+ throw new IOException("connection refused (test)");
+ },
+ 0,
+ 1);
+ CursorSendCounters shared = new CursorSendCounters();
+ shared.framesSent.set(7);
+ loop.adoptCounters(shared);
+ Assert.assertEquals("getter must read the adopted instance", 7, loop.getTotalFramesSent());
+ try {
+ loop.start();
+ try {
+ loop.adoptCounters(new CursorSendCounters());
+ Assert.fail("adoptCounters after start() must be refused");
+ } catch (IllegalStateException expected) {
+ Assert.assertTrue(expected.getMessage(), expected.getMessage().contains("before start()"));
+ }
+ } finally {
+ loop.close();
+ }
+ }
+ }
+
/**
* Regression guard for the connect-budget overflow. A large
* {@code reconnect_max_duration_millis} -- {@code Long.MAX_VALUE} is the
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SymbolDictRecycleCrashWindowsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SymbolDictRecycleCrashWindowsTest.java
new file mode 100644
index 00000000..fe9d0841
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SymbolDictRecycleCrashWindowsTest.java
@@ -0,0 +1,779 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing;
+import io.questdb.client.std.Files;
+import io.questdb.client.test.cutlass.qwp.client.QwpWireTestUtils;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT;
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Crash-window recovery for {@code QwpWebSocketSender.recycleForDictReset()}
+ * (its seven steps, quoted here for reference):
+ *
+ * What the drain + dict-order outcome below actually proves: the resume's
+ * chunk frames genuinely persisted to the SF ring and survive a
+ * close/reopen cycle, and a fresh connection replays them, in order,
+ * ahead of the data frame, against a server that strictly withholds acks
+ * for any {@code FLAG_DEFER_COMMIT} frame until the group commits -- so a
+ * malformed or missing commit that left the group open would hang the
+ * drain. It does NOT, by itself, isolate whether the resume's OWN commit
+ * closed that group -- two independent, pre-existing mechanisms
+ * (close()'s own redundant commit-closing safety net, and the recovered
+ * slot's reconnect dictionary catch-up) would produce the same drain/dict
+ * outcome even if the resume's commit were missing entirely. Isolating
+ * the resume's own commit is what the synchronous
+ * {@code hasDeferredMessagesForTesting()} assertion right after the
+ * resume runs, below, is for.
+ */
+ @Test(timeout = 60_000L)
+ public void testAbandonedResumeGroupSurvivesCrashReplay() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("crash-e-resume-rereg").toString();
+
+ // Phase 1: establish a delta baseline, kill the endpoint, abandon at
+ // CLOSE_LOOP, let the resume ring its chunks + commit, add a data
+ // frame referencing an old id, then "crash" (zero flush budget).
+ AckAllHandler phase1 = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(phase1)) {
+ // The resume's re-registration only sizes chunks when
+ // serverMaxBatchSize > 0 (see resumeRecycleIfPending); an
+ // unadvertised cap degrades to the plain baseline drop, same as
+ // full-dict mode. Advertise a generous cap so this test actually
+ // exercises the re-registration path (precedent: CloseDrainTest).
+ server.setAdvertisedMaxBatchSize(4096);
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 2L).atNow();
+ sender.flush();
+ Assert.assertTrue("setup: baseline must be established", sender.drain(5_000));
+ Assert.assertTrue(ws.getSentMaxSymbolIdForTesting() >= 0);
+
+ server.close(); // endpoint gone: everything from here stays ringed
+ ws.forceCloseLoopAbandonForTesting();
+ sender.table("t"); // resume runs here: rings chunks + commit
+ // The crash-simulated close below (flush budget 0) and a recovered
+ // delta slot's OWN reconnect catch-up (CursorWebSocketSendLoop's
+ // sentDictCount/hasReplayDictionaryDependency mirror, seeded from
+ // the persisted .symbol-dict) both independently backstop the
+ // dictionary outcome phase 2 observes below -- so drain()/dict()
+ // alone cannot pin whether the resume itself actually closed its
+ // own debt (verified by mutation: deleting resumeRecycleIfPending's
+ // sendCommitMessage() call still leaves phase 2 green, because
+ // close()'s own redundant "!deferCommit && hasDeferredMessages"
+ // safety net and the reconnect catch-up both paper over it). This
+ // synchronous check is what actually pins the resume's own success,
+ // exactly as SymbolDictRecycleTest#testCloseLoopAbandonReregistersBaseline
+ // does for the same-process case.
+ Assert.assertFalse("the resume must close its own deferred group before "
+ + "the crash window closes over it",
+ ws.hasDeferredMessagesForTesting());
+ sender.table("t").symbol("s", "a").longColumn("v", 3L).atNow(); // joins the group on the ring
+ sender.flush(); // the data frame joins them on the ring
+ } // close(): flush budget 0 -- the slot is left as a crash would leave it
+ }
+
+ // Phase 2: a fresh sender on the same slot replays the ring against a
+ // server that withholds acks for deferred frames until their commit.
+ DeferAwareCaptureHandler phase2 = new DeferAwareCaptureHandler();
+ try (TestWebSocketServer revived = startedServer(phase2)) {
+ String cfg2 = "ws::addr=localhost:" + revived.getPort() + ";sf_dir=" + sfDir + ";";
+ try (Sender replayer = Sender.fromConfig(cfg2)) {
+ Assert.assertTrue("the recovered ring -- deferred chunk group, commit, data "
+ + "frame -- must drain whole", replayer.drain(10_000));
+ }
+ Assert.assertEquals("the replayed group must re-register the old ids ahead of "
+ + "the data frames", Arrays.asList("a", "b"), phase2.dict());
+ }
+ });
+ }
+
+ /** Sorted list of entry names directly inside {@code dir} (no recursion, no "."/".."). */
+ private static ListThreading
* Handlers normally run on a dedicated daemon dispatcher thread, never on the
- * I/O thread or the producer thread. One exception: a build()-time quarantine
+ * I/O thread or the producer thread. One exception: a slot quarantine
* ({@link SenderError.Category#DATA_LOSS}) is dispatched synchronously on the
- * thread calling {@code build()} — the async dispatcher belongs to the
- * connected sender, which does not exist yet at build time. Handlers must not
- * block: for the build-time case, {@code build()} is waiting.
+ * calling thread — at {@code build()} time because the async dispatcher does
+ * not exist yet, and during a symbol-dictionary recycle rebuild (inside
+ * {@code table(...)}, on the producer thread) because a data-loss notice must
+ * not be dropped under inbox pressure. Handlers must not block or call back
+ * into the sender: in both cases the caller is waiting.
* Slow handlers cannot stall publishing; if the bounded
* inbox fills up, surplus notifications are dropped (visible via
* {@code QwpWebSocketSender.getDroppedErrorNotifications()}).
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
index 3e9fa1a3..44c01489 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
@@ -155,7 +155,11 @@ public int getOrAddSymbol(CharSequence symbol) {
+ ". Rows using already-registered symbol values continue to work. To start a fresh "
+ "dictionary, close this sender and build a new one (with store-and-forward the "
+ "buffered backlog drains first). For unbounded-cardinality data use varchar "
- + "columns instead of symbol");
+ + "columns instead of symbol. The automatic dictionary reset "
+ + "(symbol_dict_reset, symbol_dict_reset_threshold) and "
+ + "Sender.resetSymbolDictionary() avoid this cap, but both act only "
+ + "on senders created via Sender.build()/fromConfig(), and the reset "
+ + "itself runs at a table() call once the backlog is acknowledged.");
}
// Assign new ID — toString() only for new symbols that must be stored
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 41cc0a8c..15071651 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -42,6 +42,7 @@
import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer;
import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener;
import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerPool;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendCounters;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener;
@@ -138,6 +139,32 @@ public class QwpWebSocketSender implements Sender {
// Finite fallback (ms) for BACKGROUND (drainer) TCP connects when the
// user left connect_timeout unset. See effectiveConnectTimeoutMs.
public static final int DEFAULT_BACKGROUND_CONNECT_TIMEOUT_MS = 15_000;
+ // Default for symbol_dict_reset -- periodic symbol-dictionary recycling is
+ // on by default so a long-lived sender's dictionary does not grow without
+ // bound. The recycle runs at a table() call that finds the backlog
+ // acknowledged; DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS bounds how long
+ // an armed recycle may wait for that instant before one table() call
+ // pauses to drain the backlog itself (see Sender#resetSymbolDictionary()).
+ public static final boolean DEFAULT_SYMBOL_DICT_RESET_ENABLED = true;
+ // Default for symbol_dict_reset_max_wait_millis: 2 s. Once a recycle has
+ // been armed that long without a table() call finding the backlog already
+ // drained, the next table() call blocks the producing thread for up to
+ // that long waiting for the backlog to drain, then recycles. A producer
+ // that keeps a few frames in flight at every row start (a continuous
+ // stream) never exposes a drained instant on its own; on a healthy link
+ // the pause is one ack round trip, since the paused producer stops
+ // refilling the ring. On timeout the call gives up (still armed, retried
+ // only at a later drained table() call): the wait runs at most once per
+ // armed window, so an outage costs the producer one bounded pause. Kept
+ // well below QuestDBBuilder.DEFAULT_ACQUIRE_TIMEOUT_MILLIS (5 s): a
+ // pooled sender inherits an armed recycle at give-back, and a borrower
+ // paying this wait while holding its lease must release before other
+ // threads' acquire attempts expire. 0 disables the wait entirely
+ // (opportunistic-only).
+ public static final long DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS = 2_000L;
+ // Default for symbol_dict_reset_threshold: distinct-symbol count that
+ // triggers a recycle once symbol_dict_reset is on.
+ public static final int DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS = 100_000;
private static final int DEFAULT_BUFFER_SIZE = 8192;
private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB
private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class);
@@ -145,6 +172,13 @@ public class QwpWebSocketSender implements Sender {
// sf-client.md section 4.4 floor: drop-oldest under bursts needs a wide
// enough window to preserve the trailing category distribution.
private static final int MIN_ERROR_INBOX_CAPACITY = 16;
+ // Upper bound on how long recycleForDictReset step 3 waits for a DEFERRED
+ // engine close (SF worker wedged in a syscall past SegmentManager's
+ // bounded join) to release the slot flock before giving up and latching
+ // the sender terminal. Sized well past any transient disk/NFS stall the
+ // deferred-close machinery exists to survive; a worker still wedged after
+ // this long is treated as a genuinely dead disk.
+ private static final long RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS = 30_000L;
private static final String WRITE_PATH = "/write/v4";
// Yields the Authorization header value presented on each WebSocket upgrade. A constant for a
// fixed token or Basic credential; for an httpTokenProvider it pulls a freshly refreshed token,
@@ -256,10 +290,25 @@ public class QwpWebSocketSender implements Sender {
private String currentTableName;
// Cursor SF engine: the producer (user thread) writes encoded QWP frames
// into the engine's mmap'd ring; the cursorSendLoop is the I/O thread
- // that walks the ring and sends frames.
- private CursorSendEngine cursorEngine;
- private CursorWebSocketSendLoop cursorSendLoop;
+ // that walks the ring and sends frames. Both volatile since the recycle
+ // started reassigning them (non-null -> null -> non-null on the producer
+ // thread): the monitoring accessors (getAckedFsn, awaitAckedFsn, the
+ // error-check paths) read them from a monitor thread, same reasoning as
+ // symbolDictEpoch.
+ private volatile CursorSendEngine cursorEngine;
+ private volatile CursorWebSocketSendLoop cursorSendLoop;
+ // Sender-lifetime observability counters (see CursorSendCounters). Every
+ // loop generation and every attached engine adopts this instance before
+ // use, so the getTotal* accessors survive a symbol-dictionary recycle
+ // with no arithmetic at the swap. Final: the reference never moves, and
+ // monitor threads read through the AtomicLongs.
+ private final CursorSendCounters counters = new CursorSendCounters();
private boolean deferCommit;
+ // Test seam: runs once when awaitDeferredEngineClose() actually begins
+ // parking (positive witness that the await engaged rather than
+ // completing inline -- see SymbolDictRecycleDeferredCloseTest).
+ private Runnable deferredCloseParkWitness;
+ private volatile Runnable ackedFsnReadWitness;
// True when the sender emits incremental (delta) symbol dictionaries: each
// message carries only symbol ids not yet sent on the wire, rather than the
// full dictionary from id 0. Enabled in memory-mode (a reconnect replays from
@@ -295,6 +344,9 @@ public class QwpWebSocketSender implements Sender {
// while the producer thread reads it from sendRow without
// holding the sender monitor.
private volatile int effectiveAutoFlushBytes;
+ // Installed by build() once connect() succeeds; null for a sender that
+ // has never connected. See setEngineRebuildFactory.
+ private EngineRebuildFactory engineRebuildFactory;
private volatile SenderErrorDispatcher errorDispatcher;
// Async-delivery sink for SenderError notifications. Default-constructed
// here with the loud-not-silent default handler; a builder hook can swap
@@ -302,7 +354,45 @@ public class QwpWebSocketSender implements Sender {
private SenderErrorHandler errorHandler = DefaultSenderErrorHandler.INSTANCE;
private int errorInboxCapacity = SenderErrorDispatcher.DEFAULT_CAPACITY;
private long firstPendingRowTimeNanos;
+ // Additive offset applied to every user-visible FSN this sender reports
+ // (flushAndGetSequence, awaitAckedFsn's target, getAckedFsn, drain's
+ // watermark, and every FSN the I/O loop surfaces through the progress
+ // and error dispatchers). Stays 0 until a later symbol-dict recycle
+ // rebuilds the cursor engine and restarts its internal FSNs at 0 --
+ // rollFsnEpochBaseForTesting (and its production counterpart in the
+ // recycle path) advance it past every FSN already handed out, so the
+ // external sequence stays strictly monotone across the internal reset.
+ // Rule everywhere it is applied: external = fsnEpochBase + raw: raw
+ // -1 (no-data) sentinels are never translated. Volatile because the
+ // recycle rolls it while a monitor thread may be inside getAckedFsn /
+ // awaitAckedFsn: a stale base paired with a fresh engine would report
+ // an FSN dip to -1 (same reasoning as symbolDictEpoch).
+ private volatile long fsnEpochBase = 0;
private boolean hasDeferredMessages;
+ // Latched true the first time ensureConnected() completes. Once set,
+ // every later ensureConnected() -- today only the recycle's step 7 and
+ // its retry-on-next-send path -- takes the deferred (ASYNC-style)
+ // branch regardless of initialConnectMode: the store-and-forward
+ // contract scopes foreground connectivity errors to initialization
+ // only, so post-initial (re)connects belong to the I/O loop's
+ // indefinite retry (Invariant B), never to the producer thread.
+ private boolean hasInitialConnectRun;
+ // Sender-lifetime sticky OR of every rebuilt loop's own hasEverConnected:
+ // once ANY loop instance owned by this sender has reached the server,
+ // this stays true even after a symbol-dict recycle rebuilds the loop.
+ // Latched in two places -- ensureConnected()'s tail on a successful
+ // foreground (client != null) connect, and recycleForDictReset()'s step
+ // 2, which OR's in the outgoing loop's own hasEverConnected() before
+ // closing it (covers an ASYNC-initial sender whose only connect ever
+ // happened on the I/O thread, so this method never observed client !=
+ // null). ensureConnected() seeds it into the freshly built loop via
+ // markEverConnected() before start(), so a post-recycle loop rebuild
+ // does not reset CursorWebSocketSendLoop's own hasEverConnected back to
+ // false -- which would wrongly re-arm its startup-terminal
+ // classification (endpointPolicyFailureIsTerminal()) and misclassify
+ // wasEverConnected() for the whole post-recycle outage window. Volatile:
+ // wasEverConnected() is consulted from the error-dispatcher daemon.
+ private volatile boolean hasLoopEverConnected;
// FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame this session
// published, or -1 when none. Frames above it are deferred and uncommitted:
// the server withholds their acks by design (their rows are rolled back on
@@ -356,7 +446,8 @@ public class QwpWebSocketSender implements Sender {
// Engine whose close() could not complete during sender close() — its
// cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased()
// re-probes it so a late flock release becomes visible to the owning pool.
- // Only ever set inside close(); null for a sender that closed cleanly.
+ // Set by close() and by a recycle whose deferred-close await ran out; null
+ // while no engine is retained.
private volatile CursorSendEngine retainedEngine;
private int pendingRowCount;
private SenderProgressDispatcher progressDispatcher;
@@ -376,6 +467,120 @@ public class QwpWebSocketSender implements Sender {
// CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS.
private long catchUpCapGapMinEscalationWindowMillis =
CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS;
+ // Whether the producer periodically recycles (rebuilds) its symbol
+ // dictionary once resetThresholdSymbols distinct symbols have been
+ // registered, bounding unbounded dictionary growth on a long-lived sender
+ // (connect-string key symbol_dict_reset).
+ private boolean resetEnabled = DEFAULT_SYMBOL_DICT_RESET_ENABLED;
+ // Bounded wait for a starved recycle: once a recycle has been armed
+ // longer than this window without an opportunistic (idle) drain, the next
+ // table() call blocks the calling thread for up to this many millis
+ // waiting for the backlog to drain, then recycles; on timeout that call
+ // gives up (still armed, retried opportunistically later) instead of
+ // blocking further, and no second wait runs until a swap commits. 0
+ // disables the wait: the recycle then only ever runs once a row-start
+ // call (table()) finds the backlog already drained (connect-string key
+ // symbol_dict_reset_max_wait_millis; default
+ // DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS).
+ private long resetMaxWaitMillis = DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS;
+ // Distinct-symbol count that triggers a recycle once resetEnabled is on
+ // (connect-string key symbol_dict_reset_threshold).
+ private int resetThresholdSymbols = DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS;
+ // Anti-thrash floor for the automatic reset. 0 until the
+ // first swap; the effective re-arm bar is max(resetThresholdSymbols,
+ // resetFloorSymbols). Each swap raises it to twice the dictionary size
+ // at that swap, so a live symbol set larger than the threshold stops
+ // re-arming after at most ~log2(liveSet/threshold) swaps, while a
+ // genuinely unbounded-cardinality producer keeps recycling: the floor is
+ // capped at half the protocol cap so it can never double into the hard
+ // stop. Never lowered -- a shrunken working set simply stops arming, and a
+ // manual resetSymbolDictionary() swap bypasses the floor for its own swap
+ // without lowering it (max() at the commit).
+ private int resetFloorSymbols;
+ // Wall-clock time (System.nanoTime()) at which resetArmed last flipped
+ // false -> true. Recorded by armIfEligible so maybeBlockForStarvedReset's
+ // opportunistic wait can measure how long the recycle has been armed
+ // against resetMaxWaitMillis.
+ private long armedSinceNanos;
+ // Set by resetSymbolDictionary() (the public advisory API) and never
+ // cleared by armIfEligible itself -- once a caller asks for a fresh epoch,
+ // every later armIfEligible call keeps arming until the recycle actually
+ // runs and consumes the request.
+ private boolean manualResetRequested;
+ // True once armIfEligible has determined a recycle should happen. Consumed
+ // by the recycle trigger; set only from armIfEligible's two safe call
+ // points (the tail of resetTableBuffersAfterFlush, and
+ // resetSymbolDictionary() when no flush is in flight), never on the
+ // per-symbol registration path. volatile: isResetArmed() is a documented
+ // monitoring surface; a monitoring thread is its obvious reader.
+ private volatile boolean resetArmed;
+ // Cleared on the false -> true armed transition; maybeBlockForStarvedReset's
+ // opportunistic-wait step sets it once it has waited out its window for
+ // THIS arm cycle, so a subsequent forced-wait check does not re-wait.
+ private boolean starvationWaitDoneThisArm;
+ // Incremented once per completed starvation wait that timed out without
+ // the backlog draining (maybeBlockForStarvedReset's deadline branch). 0
+ // until the first such timeout. volatile: this is public API (see
+ // getSymbolDictResetStarvationTimeouts()), and a monitoring thread is
+ // its obvious reader.
+ private volatile long symbolDictResetStarvationTimeouts;
+ // External-scale FSN of the last frame proven durably acked by a recycle's
+ // barrier, recorded at recycleForDictReset step 1 before any teardown.
+ // -1 until the first recycle that had published anything. Lets the
+ // monitoring accessors (getAckedFsn, awaitAckedFsn's null-engine branch)
+ // keep reporting the durable watermark instead of collapsing to -1 while
+ // cursorEngine is transiently null mid-swap or permanently null after a
+ // failed recycle -- all pre-swap data really is acked, so the watermark
+ // stays truthful. Volatile: those accessors are exactly the surface a
+ // monitoring thread reads mid-swap, same reasoning as symbolDictEpoch.
+ private volatile long lastRecycleDurableFsn = -1L;
+ // Budget for recycleForDictReset's deferred-close await (see
+ // RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS); non-final only so tests can
+ // shrink it to drive the timeout branch.
+ private long recycleDeferredCloseMaxWaitMillis = RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS;
+ private long recycleDeferredCloseDeadlineNanos = Long.MIN_VALUE;
+ // Set (once) by completeRecycleRebuild when a rebuilt engine recovered
+ // UNACKED frames from the slot the outgoing close was supposed to have
+ // emptied -- the one failure that proves the fully-drained close contract
+ // was breached, so the producer's fresh dictionary and the slot's on-disk
+ // state have provably diverged and this sender refuses further use.
+ // checkRecycleFailure() rethrows a fresh LineSenderException wrapping this
+ // cause on every later table()/flush-family call; close() still works
+ // normally. Every other recycle failure is transient and resumable (see
+ // recycleResume), never latched here.
+ private Throwable recycleFailure;
+ // Resumable recycle: a transient failure mid-recycle no
+ // longer latches the sender terminal. CLOSE_LOOP = step 2 failed, the
+ // old loop is still dying and the old engine/dictionary are intact.
+ // REBUILD = the old engine is closed (possibly still releasing its slot
+ // flock); await/rebuild/commit are pending. resumeRecycleIfPending()
+ // advances the state from the table() barrier and ensureConnected().
+ private RecycleResume recycleResume = RecycleResume.NONE;
+ // The closed-but-not-yet-released outgoing or recovered engine a REBUILD
+ // resume still awaits; null once its deferred close completes.
+ private CursorSendEngine recyclePendingOutgoing;
+ // Raw last-published FSN of the outgoing epoch (step-1 snapshot),
+ // consumed by the commit when a REBUILD resume completes.
+ private long recyclePendingLastPublishedFsn = -1L;
+ // Test seam: recycle step-7 fault injection. When set, runs (and is
+ // expected to throw) inside ensureConnected()'s loop-construction try,
+ // after cursorSendLoop is assigned but before start() -- exercising the
+ // catch that closes and nulls the fresh loop, i.e. the failed-reconnect
+ // state SymbolDictRecycleStep7FaultTest pins.
+ private Runnable loopStartFault;
+ // Test seam: runs between the CLOSE_LOOP resume's chunk publish and its
+ // commit, so a commit-path failure is reachable deterministically.
+ @TestOnly
+ private volatile Runnable resumeCommitFaultForTesting;
+ // Test seam: runs at the top of every publishDictionaryChunk() call, so a
+ // mid-publish failure (a prefix of the chunks on the ring, the rest not)
+ // is reachable deterministically.
+ @TestOnly
+ private volatile Runnable chunkPublishFaultForTesting;
+ // Incremented once per completed symbol-dictionary recycle. 0 until the
+ // first recycle commits. volatile: this is public API (see
+ // getSymbolDictEpoch()), and a monitoring thread is its obvious reader.
+ private volatile long symbolDictEpoch;
private long reconnectInitialBackoffMillis =
CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS;
private long reconnectMaxBackoffMillis =
@@ -398,7 +603,13 @@ public class QwpWebSocketSender implements Sender {
// Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because
// the I/O thread re-registers the full dictionary via a catch-up frame before
// replaying, so the producer's delta baseline stays valid across the wire
- // boundary. Used only when deltaDictEnabled; ignored in full-dict mode.
+ // boundary. It restarts at -1 at the recycle's swap commit (fresh
+ // dictionary). The CLOSE_LOOP resume re-registers [0..sentMaxSymbolId]
+ // onto the ring and keeps it; if that publish fails part-way,
+ // publishDictionaryChunks clamps it to the chunks that reached the ring
+ // (-1 when none did), so it always equals the ringed coverage -- the
+ // invariant reclaimUnsentSymbolIds' floor relies on. Used only when
+ // deltaDictEnabled; ignored in full-dict mode.
private int sentMaxSymbolId = -1;
// When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only
// explicit flush() triggers the server-side commit. Enables accumulating
@@ -804,7 +1015,10 @@ public static QwpWebSocketSender connectWithCredentialSupplier(
connectionListener, connectionListenerInboxCapacity,
CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS,
- CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
+ CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS,
+ DEFAULT_SYMBOL_DICT_RESET_ENABLED,
+ DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS,
+ DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS);
}
/**
@@ -816,7 +1030,9 @@ public static QwpWebSocketSender connectWithCredentialSupplier(
*
+ *
+ * This method runs steps 1-3 and hands steps 4-7 to
+ * {@link #completeRecycleRebuild(int, long)}. The producer-visible swap
+ * (dictionary, epoch) commits only once a fresh engine stands on
+ * the emptied slot, and a throw before that point no longer kills the
+ * sender: every frame that existed before this call was already proven
+ * acked, so nothing is at risk, and the recycle simply records how far it
+ * got ({@link #recycleResume}) and resumes from the next
+ * {@link #table(CharSequence)} or {@link #ensureConnected()} -- see
+ * {@link #resumeRecycleIfPending()}. An {@link Error} (OOM/SOE/linkage)
+ * passes through untouched, neither recorded nor wrapped -- it is not a
+ * recycle verdict.
+ *
+ *
+ * Both tests doctor a slot directly at the engine level and then point a live
+ * sender's rebuild factory at it, so the verdict is driven by real on-disk
+ * state rather than by a mocked engine.
+ *
+ * 1. lastPublishedFsn = cursorEngine.publishedFsn()
+ * 2. close cursorSendLoop (I/O thread + client)
+ * 3. cursorEngine.close() -- FULLY DRAINED (the barrier only fires the swap once
+ * isRingDrained() is true), so this unlinks every *.sfa, the ack watermark,
+ * the persisted dictionary and the logical slot lock, leaving the slot empty;
+ * a deferred close is awaited before step 4
+ * 4. cursorEngine = engineRebuildFactory.rebuild() -- a brand-new CursorSendEngine
+ * on the now-empty slot (fresh .lock/.ack-watermark/.symbol-dict/segments)
+ * 5. rollFsnEpochBase(lastPublishedFsn)
+ * 6. producer state swap: fresh GlobalSymbolDictionary, sentMaxSymbolId=-1,
+ * symbolDictEpoch++, resetArmed=false
+ * 7. reconnect (ensureConnected())
+ *
+ * This suite pins what a restarted sender recovers if the process dies at each
+ * of five points around that sequence:
+ *
+ *
+ *
+ * Why these are simulated, not paused mid-sequence
+ * {@code recycleForDictReset()} runs synchronously inside one {@code table()}
+ * call with no external hook between its steps, so a test cannot literally
+ * suspend a live sender between step 3 and step 4. And unlike
+ * {@code CursorSendEngineCrashConsistencyTest}'s bare {@code CursorSendEngine} +
+ * fault-injecting {@code FilesFacade}, a {@code Sender} built through the public
+ * API (as production always does) has no seam for a custom {@code FilesFacade}
+ * -- {@code LineSenderBuilder.constructEngineOnSlot} always goes through the real
+ * filesystem. Each arm below instead constructs the exact on-disk image a crash
+ * at that point would leave, using only real production code paths plus
+ * filesystem-level fixtures already established elsewhere in this test suite
+ * ({@code DeltaDictRecoveryTest}'s {@code writeAckWatermark}, {@code
+ * RecoveryReplayTest}'s close-fast-with-a-silent-server idiom):
+ *
+ *
+ *
+ * (b) and (c) are NOT the same recoverable state
+ * Both look empty of data and both replay nothing, but they are not
+ * byte-identical on disk, and a restarted engine can tell them apart. (b)'s
+ * directory holds nothing this engine ever created -- no manifest, no segment.
+ * (The crashed sender's own fully-drained close already removed {@code
+ * sf-manifest.bin} along with the last segment, so recovery finds NO {@code
+ * .sfa} files and NO manifest, and falls straight through to {@code
+ * Recovery.empty()}.) (c)'s directory holds the fresh rebuild's own
+ * {@code sf-manifest.bin} (boundaries collapsed at 0) and its zero-frame
+ * {@code sf-initial.sfa} / {@code sf-...0000.sfa} pair. {@code
+ * SegmentRing.recover()}'s manifest branch (the {@code chain.size() == 0}
+ * check) accepts a manifest whose {@code headBase == activeBase} alongside a
+ * same-based, zero-frame active segment as a RECOVERED (if empty) chain -- a
+ * different branch entirely from the one arm (b) falls through to. So {@code
+ * wasRecoveredFromDisk()} comes back {@code false} for (b) and {@code true} for
+ * (c): the pinned, distinguishing observable between the two, asserted
+ * explicitly below instead of writing two assertion-for-assertion duplicate
+ * tests.
+ *
+ * Oracle
+ * Every arm asserts the same three things about the RECOVERED sender: it keeps
+ * ingesting after recovery; the symbols it and its predecessor registered are
+ * exactly and correctly reconstructable from the wire (each fresh server
+ * handler rebuilds the per-connection delta dictionary via {@link
+ * QwpWireTestUtils#accumulateDeltaDictionary}); and no data (table-carrying)
+ * frame is delivered more than the at-least-once contract allows (each handler
+ * also counts data frames, so a spurious re-send shows up as an unexpected
+ * count).
+ */
+public class SymbolDictRecycleCrashWindowsTest {
+
+ /**
+ * The exact file set a freshly-rebuilt (never-flushed) engine's own slot
+ * holds -- matches {@code SymbolDictRecycleTest#testPostRecycleSlotContents}'s
+ * {@code freshSlotFiles}. Shared by arm (c)'s pre-snapshot wait and its
+ * post-restore assertion so the two can never drift apart.
+ */
+ private static final List