From de86197b3fb19d9dc274408ae08f7967b468e456 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 02:45:30 +0100 Subject: [PATCH 01/78] Delta-encode the QWP symbol dictionary Previously every QWP ingress message re-sent the entire symbol dictionary, so a connection with many distinct symbols paid to retransmit the whole dictionary on every message. The client now sends each symbol id to the server only once per connection. Memory mode: - The producer keeps a monotonic "sent" watermark and each frame carries only the ids above it (a delta section), instead of the full dictionary from id 0. - On reconnect or failover the fresh server has an empty dictionary, so the I/O thread replays the whole dictionary as a catch-up frame before any post-reconnect traffic, keeping the producer's monotonic baseline valid across the wire boundary. Store-and-forward (file mode): - Each slot persists its dictionary to a dot-prefixed side-file (PersistedSymbolDict) using write-ahead ordering: new symbols are appended before the referencing frame is published, so a recovered or orphan-drained slot on a fresh process can always rebuild the dictionary that a delta frame references. - The persistence does not fsync, matching the rest of store-and-forward, which is process-crash durable (the page cache survives) but not host-crash durable. A host crash that tears the dictionary is caught at replay by a guard that fails the send cleanly ("resend required") instead of transmitting a gapped frame that would corrupt the table. Catch-up split: - The reconnect/recovery catch-up splits across as many frames as the server's advertised batch cap requires, so a dictionary larger than the cap is re-registered without any single frame exceeding it. The frames carry contiguous id ranges and reassemble on the server exactly as the original per-frame deltas would. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 144 ++++++- .../client/sf/cursor/CursorSendEngine.java | 70 +++- .../sf/cursor/CursorWebSocketSendLoop.java | 308 +++++++++++++- .../client/sf/cursor/PersistedSymbolDict.java | 379 ++++++++++++++++++ .../qwp/client/DeltaDictCatchUpTest.java | 325 +++++++++++++++ .../qwp/client/DeltaDictRecoveryTest.java | 352 ++++++++++++++++ .../cutlass/qwp/client/ReconnectTest.java | 7 +- .../qwp/client/SelfSufficientFramesTest.java | 151 +++++-- .../sf/cursor/PersistedSymbolDictTest.java | 219 ++++++++++ .../qwp/websocket/TestWebSocketServer.java | 17 + 10 files changed, 1910 insertions(+), 62 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..286596c4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -46,6 +46,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; @@ -221,6 +222,14 @@ public class QwpWebSocketSender implements Sender { private CursorSendEngine cursorEngine; private CursorWebSocketSendLoop cursorSendLoop; private boolean deferCommit; + // True when the sender emits incremental (delta) symbol dictionaries: each + // message carries only symbol ids not yet sent on the wire, rather than the + // full dictionary from id 0. Enabled only in memory-mode, where a reconnect + // replays from the in-process ring and the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying. File-mode store-and-forward + // keeps full self-sufficient frames (recovery/orphan-drain can replay mid-stream + // to a fresh server, where a partial delta would leave gaps). Set in setCursorEngine. + private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after // startOrphanDrainers) and read at pool-creation time. Null -> drainers @@ -313,6 +322,12 @@ public class QwpWebSocketSender implements Sender { // beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the // first round indicates the initial connect. private long roundSeq; + // Highest global symbol id the producer has baked into a frame so far, or -1. + // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because + // the I/O thread re-registers the full dictionary via a catch-up frame before + // replaying, so the producer's delta baseline stays valid across the wire + // boundary. Used only when deltaDictEnabled; ignored in full-dict mode. + private int sentMaxSymbolId = -1; // When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only // explicit flush() triggers the server-side commit. Enables accumulating // arbitrarily large datasets that exceed the server's recv buffer. @@ -2215,6 +2230,18 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } this.cursorEngine = engine; this.ownsCursorEngine = takeOwnership && engine != null; + // Delta encoding is available in memory-mode (in-process catch-up) and in + // file-mode when the persisted dictionary opened (recovery / orphan-drain + // rebuild the dictionary from it). Otherwise fall back to full self- + // sufficient frames. See CursorSendEngine.isDeltaDictEnabled. + this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled(); + // Recovery: repopulate the producer's global dictionary from the slot's + // persisted dictionary so newly ingested symbols continue from the + // recovered ids (rather than colliding with them at 0), and the delta + // baseline resumes where the crashed session left off. + if (deltaDictEnabled && engine.wasRecoveredFromDisk()) { + seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); + } } /** @@ -3423,14 +3450,15 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient: - // recorded frames replay to fresh server connections (orphan-slot - // drainers and post-reconnect replay), so always emit the full - // symbol-dict delta from id=0 and the full column schema inline, - // never a back-reference the target server may not have seen. + // In full-dict mode every frame is self-sufficient: it carries the whole + // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh + // server never dangles a symbol id. In delta mode (memory-mode only) each + // frame carries only ids above sentMaxSymbolId; a reconnect re-registers + // the dictionary via an I/O-thread catch-up frame before replay, so the + // producer's monotonic baseline stays valid across the wire boundary. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3456,10 +3484,18 @@ private void flushPendingRows(boolean deferCommit) { return; } + // Write-ahead: durably persist this frame's new symbols BEFORE it is + // published, so a recovered/orphan-drained slot can always rebuild the + // dictionary the (non-self-sufficient) delta frame references. No-op in + // memory mode and when the frame introduces no new symbols. + persistNewSymbolsBeforePublish(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring; + // advance the delta baseline so the next frame ships only newer ids. + advanceSentMaxSymbolId(); hasDeferredMessages = deferCommit; if (!deferCommit) { @@ -3514,8 +3550,12 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm boolean deferThis = deferCommit || !isLast; encoder.setDeferCommit(deferThis); + // Each split frame emits the delta above sentMaxSymbolId; the first + // frame ships the whole batch's new ids and advances the baseline, so + // the remaining frames carry an empty delta and just reference ids the + // first frame already registered. encoder.beginMessage(1, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); encoder.addTable(tableBuffer); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); @@ -3528,11 +3568,18 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); } + // Write-ahead persist before publish (see flushPendingRows). The + // first split frame carries the batch's new symbols; the rest are + // no-ops once the baseline has advanced past them. + persistNewSymbolsBeforePublish(); ensureActiveBufferReady(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // Frame queued: advance so the next split frame's delta starts above + // the ids this one just registered. + advanceSentMaxSymbolId(); } encoder.setDeferCommit(false); @@ -3573,8 +3620,10 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); + // A commit carries no rows and no new symbols; in delta mode its empty + // delta simply starts at the server's current dictionary size. encoder.beginMessage(0, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); @@ -3586,15 +3635,84 @@ private void sendCommitMessage() { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + /** + * Advances the delta baseline once a frame carrying the current batch's + * symbols has been queued onto the ring. No-op in full-dict mode. Only ever + * moves the baseline forward, so a batch that used no new symbols leaves it + * unchanged. + */ + private void advanceSentMaxSymbolId() { + if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) { + sentMaxSymbolId = currentBatchMaxSymbolId; + } + } + + /** + * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 .. + * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the + * frame is published to the ring. This write-ahead ordering keeps the + * persisted dictionary a superset of every process-crash-recoverable frame's + * references, so recovery and orphan-drain can re-register it on a fresh + * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it + * is caught by the send loop's replay guard. No-op in memory mode (no + * persisted dictionary) and when the frame introduces no new symbols. + */ + private void persistNewSymbolsBeforePublish() { + if (!deltaDictEnabled || cursorEngine == null) { + return; + } + PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict(); + if (pd == null) { + return; + } + int from = sentMaxSymbolId + 1; + int to = currentBatchMaxSymbolId; + if (to < from) { + return; + } + for (int id = from; id <= to; id++) { + pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); + } + } + private void resetSymbolDictStateForNewConnection() { - // The new server has an empty symbol dictionary, so the next batch - // must ship a delta starting at id 0. beginMessage() always passes - // confirmedMaxId = -1; resetting the batch watermark here keeps a - // stale value from suppressing re-emission of symbol ids the new - // server has never seen. + // Runs on the foreground (initial) connect only -- NOT on the I/O thread's + // reconnect/failover path. The per-batch watermark is drained state, so + // clearing it here is harmless. sentMaxSymbolId is deliberately left + // untouched: in delta mode the I/O thread re-registers the whole + // dictionary with a catch-up frame on reconnect, so the producer's + // monotonic baseline must survive the wire boundary; resetting it would + // desync the producer from the I/O thread's sent-dictionary count. currentBatchMaxSymbolId = -1; } + /** + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} from + * the slot's persisted dictionary (ids assigned in the same ascending order, + * so they match the recovered frames) and resumes the delta baseline at the + * recovered tip, so newly ingested symbols continue above the recovered ids. + */ + private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + if (pd == null || pd.size() == 0) { + return; + } + ObjList symbols = pd.readLoadedSymbols(); + for (int i = 0, n = symbols.size(); i < n; i++) { + globalSymbolDictionary.getOrAddSymbol(symbols.getQuick(i)); + } + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + } + + /** + * The symbol id below which the server already holds every dictionary entry, + * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is + * the producer's monotonic sent watermark; in full-dict mode it is -1 so every + * frame re-ships the dictionary from id 0. + */ + private int symbolDeltaBaseline() { + return deltaDictEnabled ? sentMaxSymbolId : -1; + } + private void rollbackRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..85015a73 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -110,6 +110,13 @@ public final class CursorSendEngine implements QuietCloseable { // in the constructor, closed by {@link #close()}. The segment manager // writes through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; + // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null} + // in memory mode and if open() failed). Enables delta-encoded SF frames: + // recovery / orphan-drain load it to re-register the dictionary on the fresh + // server before replaying non-self-sufficient frames. Opened in the + // constructor, closed by {@link #close()}. When null in disk mode the engine + // reports delta encoding as unavailable and the sender keeps full-dict frames. + private final PersistedSymbolDict persistedSymbolDict; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -199,6 +206,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // reference instead of orphaning the mmap'd segments + fds. SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; + PersistedSymbolDict persistedDictInProgress = null; try { // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine @@ -257,6 +265,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. watermarkInProgress = AckWatermark.open(sfDir); + // Load the persisted symbol dictionary so delta-encoded frames + // in this recovered slot can be re-registered on the fresh + // server before replay. Null on open failure -> delta disabled. + persistedDictInProgress = PersistedSymbolDict.open(sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -309,6 +321,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); + // Same stale-side-file hygiene for the symbol dictionary: a + // fresh slot starts with an empty dictionary. + PersistedSymbolDict.removeOrphan(sfDir); + persistedDictInProgress = PersistedSymbolDict.open(sfDir); } MmapSegment initial; String initialPath = null; @@ -333,10 +349,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man manager.start(); } manager.register(ringInProgress, sfDir, watermarkInProgress); - // All construction succeeded — commit the ring and - // watermark references. + // All construction succeeded — commit the ring, watermark and + // symbol-dictionary references. this.ring = ringInProgress; this.watermark = watermarkInProgress; + this.persistedSymbolDict = persistedDictInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -362,6 +379,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (persistedDictInProgress != null) { + try { + persistedDictInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -541,6 +564,12 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (persistedSymbolDict != null) { + try { + persistedSymbolDict.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -550,6 +579,11 @@ public synchronized void close() { AckWatermark.removeOrphan(sfDir); } catch (Throwable ignored) { } + try { + // Slot fully drained: the dictionary has no frames behind it. + PersistedSymbolDict.removeOrphan(sfDir); + } catch (Throwable ignored) { + } } } finally { if (slotLock != null) { @@ -586,6 +620,38 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } + /** + * True when this engine has no store-and-forward directory: the ring lives + * only in malloc'd memory, so it cannot be recovered after a crash and no + * orphan drainer ever replays it. Only in-process reconnect/failover replays + * its frames, which is what makes send-time symbol-dict catch-up (rather than + * fully self-sufficient frames) safe. + */ + public boolean isMemoryMode() { + return sfDir == null; + } + + /** + * Whether the sender may delta-encode symbol dictionaries on this engine. + * Always true in memory mode (the send loop keeps an in-process catch-up + * mirror). In disk mode it requires the persisted dictionary to have opened, + * since delta frames are not self-sufficient and recovery / orphan-drain must + * be able to rebuild the dictionary from disk. When false in disk mode the + * sender falls back to full self-sufficient frames. + */ + public boolean isDeltaDictEnabled() { + return sfDir == null || persistedSymbolDict != null; + } + + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..cbf948b0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -36,9 +36,12 @@ import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; @@ -177,6 +180,20 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); + // Delta symbol dictionary catch-up state (memory-mode only; see swapClient). + // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every + // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in + // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that + // on reconnect it can re-register the whole dictionary on the fresh server + // (which discards its dictionary on every disconnect) before replaying frames + // whose deltas start above id 0. All of this is touched only by the I/O thread. + private final boolean deltaDictEnabled; + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + private int sentDictCount; + // End position (native address) written by the last readVarintAt() call. + private long varintEnd; private final CountDownLatch shutdownLatch = new CountDownLatch(1); private final AtomicLong totalAcks = new AtomicLong(); // Counters for observability of the durable-ack path. Both are zero @@ -489,6 +506,23 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + this.deltaDictEnabled = engine.isDeltaDictEnabled(); + // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, + // so seed it from the slot's persisted dictionary. That way the very first + // connection re-registers the whole dictionary (via a catch-up frame) + // before replaying the recovered delta frames. + if (deltaDictEnabled) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + if (pd != null && pd.size() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + ensureSentDictCapacity(len); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + sentDictBytesLen = len; + } + sentDictCount = pd.size(); + } + } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; @@ -1669,6 +1703,14 @@ private void ioLoop() { // best-effort } } + // The symbol-dict mirror is I/O-thread-owned; free it here, on the + // owning thread's exit path, after the last send that could touch it. + if (sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + } shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -1849,8 +1891,6 @@ private void swapClient(WebSocketClient newClient) { // past the tail instead of replaying into it. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; // Snapshot publishedFsn at swap time — frames at FSN ≤ this value // were already on the wire before the drop and will be replayed. // trySendOne resets replayTargetFsn to -1 once we cross the boundary. @@ -1862,9 +1902,240 @@ private void swapClient(WebSocketClient newClient) { // carrying stale state across the wire boundary would either // double-trim or starve the queue. clearDurableAckTracking(); + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } + /** + * Sets the wire-sequence baseline for a fresh connection and, when the symbol + * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first + * so the fresh server (whose dictionary starts empty) can resolve the + * non-self-sufficient delta frames that replay next. + *

+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just + * below {@code replayStart} (a harmless re-ack); real replay frames then follow + * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode), + * or before a client exists (async initial connect), keep the plain 1:1 + * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the + * first real connection via swapClient. + */ + private void setWireBaselineWithCatchUp(long replayStart) { + if (client != null && deltaDictEnabled && sentDictCount > 0) { + this.nextWireSeq = 0L; + // The catch-up may span several frames when the dictionary exceeds the + // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps + // to an already-acked FSN, so the first real frame still lands on + // replayStart. + int catchUpFrames = sendDictCatchUp(); + this.fsnAtZero = replayStart - catchUpFrames; + } else { + this.fsnAtZero = replayStart; + this.nextWireSeq = 0L; + } + } + + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + */ + /** + * Returns the symbol-dictionary delta start id of a frame, or -1 when the + * frame carries no delta section. Used by the pre-send torn-dictionary guard. + */ + private int frameDeltaStart(long payloadAddr, int payloadLen) { + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return -1; + } + return (int) readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen); + } + + // True only for a well-formed QWP frame this encoder produced that carries a + // delta symbol-dict section. The magic check keeps the dict logic from + // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose + // bytes happen to set the delta flag. + private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + } + + private void accumulateSentDict(long payloadAddr, int payloadLen) { + long limit = payloadAddr + payloadLen; + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return; + } + long p = payloadAddr + QwpConstants.HEADER_SIZE; + long deltaStart = readVarintAt(p, limit); + p = varintEnd; + long deltaCount = readVarintAt(p, limit); + p = varintEnd; + // Only a delta that extends exactly from our current tip introduces new + // symbols. deltaStart < sentDictCount is a replay/overlap we already hold; + // deltaStart > sentDictCount is rejected before send by the torn-dictionary + // guard in trySendOne, so it never reaches here. + if (deltaCount <= 0 || deltaStart != sentDictCount) { + return; + } + long regionStart = p; + for (long i = 0; i < deltaCount; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + // Malformed -- never happens for frames we encoded; bail rather + // than corrupt the mirror. + return; + } + } + int regionBytes = (int) (p - regionStart); + ensureSentDictCapacity(sentDictBytesLen + regionBytes); + Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); + sentDictBytesLen += regionBytes; + sentDictCount += (int) deltaCount; + } + + private void ensureSentDictCapacity(int required) { + if (sentDictBytesCapacity >= required) { + return; + } + int newCap = Math.max(sentDictBytesCapacity * 2, Math.max(4096, required)); + sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); + sentDictBytesCapacity = newCap; + } + + private long readVarintAt(long p, long limit) { + long value = 0; + int shift = 0; + long cur = p; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(cur++); + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + varintEnd = cur; + return value; + } + + /** + * Builds a table-less catch-up frame carrying the full symbol dictionary + * ({@code deltaStart=0, deltaCount=sentDictCount}) and sends it to the freshly + * connected server, then consumes wire seq 0 for it. {@code sendBinary} copies + * the bytes synchronously, so the temporary frame buffer is freed immediately. + * On send failure it fails the loop, which recycles and retries on the next + * connection. + */ + /** + * Re-registers the whole symbol dictionary on a fresh connection, split into + * as many table-less frames as the server's advertised batch cap requires so + * no single frame exceeds it (a large dictionary would otherwise be rejected). + * Each chunk carries a contiguous id range {@code [start .. start+count)}, in + * order, so the server accumulates them exactly as it would the original + * per-frame deltas. Returns the number of frames sent (each consumed a wire + * sequence), so the caller can align {@code fsnAtZero}. Stops early and fails + * the loop on a send error or an entry too large for the cap. + */ + private int sendDictCatchUp() { + int cap = client.getServerMaxBatchSize(); + // Symbol-bytes budget per frame, leaving room for the 12-byte header and + // the two delta-section varints. cap <= 0 means the server advertised no + // limit -> send the whole dictionary in one frame. + int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : Integer.MAX_VALUE; + int framesSent = 0; + int chunkStartId = 0; + long chunkStartAddr = sentDictBytesAddr; + int chunkSymbols = 0; + long chunkBytes = 0; + long p = sentDictBytesAddr; + long limit = sentDictBytesAddr + sentDictBytesLen; + while (p < limit) { + long entryStart = p; + long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix + long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] + long entryBytes = entryEnd - entryStart; + if (entryBytes > budget) { + fail(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "entryBytes=" + entryBytes + ", budget=" + budget + ']')); + return framesSent; + } + if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { + if (!sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + return framesSent; + } + framesSent++; + chunkStartId += chunkSymbols; + chunkStartAddr = entryStart; + chunkSymbols = 0; + chunkBytes = 0; + } + chunkSymbols++; + chunkBytes += entryBytes; + p = entryEnd; + } + if (chunkSymbols > 0 && sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + framesSent++; + } + return framesSent; + } + + /** + * Sends one table-less catch-up frame carrying dictionary ids + * {@code [deltaStart .. deltaStart+deltaCount)}. Returns false and fails the + * loop on a send error. + */ + private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + int payloadLen = NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + symbolsLen; + int frameLen = QwpConstants.HEADER_SIZE + payloadLen; + long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); + Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); + Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); + Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); + Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT)); + Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount + Unsafe.getUnsafe().putInt(frame + 8, payloadLen); + long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); + q = writeVarintAt(q, deltaCount); + Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); + client.sendBinary(frame, frameLen); + } catch (Throwable t) { + fail(t); + return false; + } finally { + Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); + } + nextWireSeq++; // this catch-up chunk consumed a wire sequence + lastFrameOrPingNanos = System.nanoTime(); + totalFramesSent.incrementAndGet(); + return true; + } + + private static long writeVarintAt(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + private boolean tryReceiveAcks() { boolean any = false; try { @@ -1949,12 +2220,38 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + if (deltaDictEnabled) { + // Torn-dictionary guard. In normal operation a delta frame's start id + // never exceeds the dictionary coverage established so far (replayed + // frames overlap the catch-up dict; fresh frames extend it + // contiguously). A gap here means the persisted dictionary was torn -- + // almost always by a host/power crash, which leaves segment frames on + // disk but loses recently-written dictionary entries (SF, like the rest + // of the store, is process-crash durable but not host-crash durable). + // Sending the frame would corrupt the table (the server would null-pad + // the missing ids), so fail terminally instead; the unreplayable data + // must be resent. + int deltaStart = frameDeltaStart(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; + } + } try { client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); } catch (Throwable t) { fail(t); return false; } + if (deltaDictEnabled) { + // Mirror the symbols this frame introduced so a later reconnect can + // rebuild the whole dictionary. Idempotent on replay: a frame whose + // delta we already hold advances nothing. + accumulateSentDict(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; @@ -1984,8 +2281,11 @@ void positionCursorForStart() { // starts past it. Zero wire cost, no recycle. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; + // Recovery / orphan-drain seed the dictionary mirror, so the initial + // connection may also need a catch-up (client is non-null in the + // sync-start and drainer paths; null in async-initial, where swapClient + // handles it on the first connect). + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java new file mode 100644 index 00000000..4c2674b9 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -0,0 +1,379 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Append-only, per-slot persistence of the global symbol dictionary that a + * store-and-forward sender ships to the server with delta encoding. Lives at + * {@code /.symbol-dict} alongside the segment files, the slot lock and + * {@code .ack-watermark}. + *

+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the + * symbols it introduces, so recovering (process restart) or draining (orphan + * adoption) a slot requires re-registering the whole dictionary on the fresh + * server before those frames replay. This file is that dictionary. Unlike + * {@link AckWatermark} -- a discardable optimization protected by a + * {@code max()} clamp -- this file is load-bearing: a surviving frame + * that references an id missing from it is unrecoverable. It is therefore held + * to a stronger durability contract. + *

+ * Layout (little-endian): + * 

+ *   offset 0: u32 magic = 'SYD1'
+ *   offset 4: u8  version = 1
+ *   offset 5: 3 bytes reserved (zero)
+ *   offset 8: entries, each [len: varint][utf8 bytes], in ascending global-id order
+ *
+ * Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned + * sequentially from 0), so no id needs to be stored. + *

+ * Durability / write-ahead ordering: the producer appends the symbols a + * frame introduces BEFORE that frame is published to the ring, but does NOT + * fsync -- matching the rest of store-and-forward, which is page-cache (not + * disk) durable. This ordering is sufficient for a process/JVM crash: the + * page cache survives, so both the dictionary and the frames survive and the + * dictionary is a superset of every recoverable frame's references. It is NOT + * sufficient for a host/power crash, where unflushed pages can be lost out + * of order and the dictionary may end up torn relative to the frames it serves -- + * exactly as the segment frames themselves may be lost on a host crash. A torn + * dictionary is caught at replay by the send loop's guard, which fails loudly + * (the unreplayable data must be resent) rather than corrupting the target table. + *

+ * A torn trailing entry from a crash mid-append is self-healing: {@link #open} + * stops parsing at the first incomplete entry and the next append overwrites it. + *

+ * Lifecycle: single-writer (the producer / user thread). Read once at + * {@link #open} to seed in-memory state on recovery or orphan-drain. Owner + * (the engine) closes it. Not thread-safe for concurrent writers. + */ +public final class PersistedSymbolDict implements QuietCloseable { + + /** + * Filename within the slot directory. Dot-prefixed so directory + * enumerators that filter by the {@code .sfa} suffix (segment recovery, + * OrphanScanner, trim) skip it automatically. + */ + public static final String FILE_NAME = ".symbol-dict"; + static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian + static final int HEADER_SIZE = 8; + static final byte VERSION = 1; + // Guards against a hostile/corrupt varint length driving a huge allocation + // or a runaway parse. Symbols are short; this is a generous ceiling. + private static final int MAX_ENTRY_LEN = 1 << 20; + private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); + private final int fd; + // In-memory copy of the entry region [len][utf8]... exactly as on disk, + // populated only when open() recovered existing entries (recovery / + // orphan-drain). Zero/empty for a freshly created file. Consumed once to + // seed the send loop's catch-up mirror and the producer's id map. + private final long loadedEntriesAddr; + private final int loadedEntriesLen; + private long appendOffset; + private boolean closed; + private long scratchAddr; + private int scratchCap; + private int size; + + private PersistedSymbolDict(int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { + this.fd = fd; + this.appendOffset = appendOffset; + this.size = size; + this.loadedEntriesAddr = loadedEntriesAddr; + this.loadedEntriesLen = loadedEntriesLen; + } + + /** + * Opens (creating if absent) the dictionary file in {@code slotDir}. An + * existing file is parsed and its complete entries are loaded into memory + * (see {@link #loadedEntriesAddr()}); a missing or invalid file is (re)created + * with a fresh header. Returns {@code null} on any I/O failure -- the caller + * then falls back to full-dictionary (self-sufficient) frames for this slot, + * so a broken side-file degrades gracefully rather than aborting the sender. + */ + public static PersistedSymbolDict open(String slotDir) { + String filePath = slotDir + "/" + FILE_NAME; + long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + if (existing >= HEADER_SIZE) { + PersistedSymbolDict d = openExisting(filePath, existing); + if (d != null) { + return d; + } + // Fall through to a clean re-create: a header/parse failure on an + // existing file means it cannot be trusted for delta replay. + } + return openFresh(filePath); + } + + /** + * Best-effort removal of a stale dictionary file. Used at fresh-start (a + * stale dict with no segments behind it is meaningless) and at fully-drained + * close (the slot is empty, nothing references the dictionary any more), + * mirroring {@link AckWatermark#removeOrphan}. + */ + public static void removeOrphan(String slotDir) { + Files.remove(slotDir + "/" + FILE_NAME); + } + + /** + * Appends one symbol, extending the on-disk dictionary. The caller appends a + * frame's new symbols BEFORE publishing that frame, so the write ordering + * (dictionary entry before referencing frame) holds; no fsync is performed + * (see the class-level durability note). Assigns the next dense id implicitly + * (the entry's position). + */ + public void appendSymbol(CharSequence symbol) { + if (closed) { + return; + } + int utf8Len = Utf8s.utf8Bytes(symbol); + int varLen = varintSize(utf8Len); + int recLen = varLen + utf8Len; + ensureScratch(recLen); + long p = writeVarint(scratchAddr, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + long written = Files.write(fd, scratchAddr, recLen, appendOffset); + if (written != recLen) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + recLen + ", actual=" + written + ']'); + } + appendOffset += recLen; + size++; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + if (loadedEntriesAddr != 0L) { + Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT); + } + if (scratchAddr != 0L) { + Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); + scratchAddr = 0L; + scratchCap = 0; + } + if (fd >= 0) { + Files.close(fd); + } + } + + /** + * Base address of the loaded entry region -- the concatenated + * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly + * as a delta section carries them. Zero when nothing was recovered. + */ + public long loadedEntriesAddr() { + return loadedEntriesAddr; + } + + /** + * Byte length of {@link #loadedEntriesAddr()}. + */ + public int loadedEntriesLen() { + return loadedEntriesLen; + } + + /** + * Materialises the loaded entries as symbol strings in ascending-id order + * (entry {@code i} is symbol id {@code i}). Used once on recovery to + * repopulate the producer's global dictionary. Empty when nothing was + * recovered. + */ + public ObjList readLoadedSymbols() { + ObjList out = new ObjList<>(Math.max(size, 1)); + long p = loadedEntriesAddr; + long limit = p + loadedEntriesLen; + for (int i = 0; i < size && p < limit; i++) { + long len = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + len |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + if (p + len > limit) { + break; // defensive: torn tail (should not happen past parse in open) + } + out.add(Utf8s.stringFromUtf8Bytes(p, p + len)); + p += len; + } + return out; + } + + /** + * Number of symbols the dictionary holds (highest id + 1). + */ + public int size() { + return size; + } + + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { + int fd = Files.openRW(filePath); + if (fd < 0) { + LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd); + return null; + } + long buf = 0L; + try { + int len = (int) fileLen; + buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + long read = Files.read(fd, buf, len, 0); + if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC) { + LOG.warn("symbol dict {} unreadable or bad magic; recreating", filePath); + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + return null; + } + // Parse complete entries starting after the header; stop at the + // first torn/incomplete trailing entry. + int pos = HEADER_SIZE; + int count = 0; + while (pos < len) { + long[] vr = decodeVarint(buf, pos, len); + if (vr == null) { + break; // torn length varint + } + int entryLen = (int) vr[0]; + int next = (int) vr[1]; + if (entryLen < 0 || entryLen > MAX_ENTRY_LEN || (long) next + entryLen > len) { + break; // torn/oversized entry -- self-healing tail + } + pos = next + entryLen; + count++; + } + int entriesLen = pos - HEADER_SIZE; + long entriesAddr = 0L; + if (entriesLen > 0) { + entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); + } + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + buf = 0L; + // appendOffset lands just past the last complete entry, so the next + // append overwrites any torn trailing bytes. + return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); + } catch (Throwable t) { + if (buf != 0L) { + Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); + } + Files.close(fd); + LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); + return null; + } + } + + private static PersistedSymbolDict openFresh(String filePath) { + int fd = Files.openCleanRW(filePath); + if (fd < 0) { + LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); + return null; + } + long hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC); + Unsafe.getUnsafe().putByte(hdr + 4, VERSION); + Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); + Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0); + Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0); + long written = Files.write(fd, hdr, HEADER_SIZE, 0); + if (written != HEADER_SIZE) { + Files.close(fd); + Files.remove(filePath); + LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); + return null; + } + } finally { + Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + } + return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); + } + + /** + * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns + * {@code [value, newPos]} or {@code null} if the varint is truncated + * (torn tail). + */ + private static long[] decodeVarint(long buf, int pos, int limit) { + long value = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return new long[]{value, cur}; + } + shift += 7; + if (shift > 35) { + return null; // implausible for an entry length; treat as torn + } + } + return null; + } + + private static int varintSize(long value) { + int n = 1; + while (value > 0x7F) { + value >>>= 7; + n++; + } + return n; + } + + private static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + + private void ensureScratch(int required) { + if (scratchCap >= required) { + return; + } + int newCap = Math.max(required, Math.max(256, scratchCap * 2)); + scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, newCap, MemoryTag.NATIVE_DEFAULT); + scratchCap = newCap; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java new file mode 100644 index 00000000..5e9af978 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -0,0 +1,325 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode). + *

+ * When a memory-mode sender reconnects, the server it lands on has an empty + * dictionary (the server discards it on every disconnect). Because the producer + * ships monotonic deltas -- each symbol id once -- a naive replay would leave the + * fresh server with a dictionary gap. The I/O thread prevents this by sending a + * full-dictionary catch-up frame before any post-reconnect traffic. This test + * reconstructs the server's per-connection dictionary from the captured wire + * bytes and asserts it stays complete and gap-free across the reconnect. + */ +public class DeltaDictCatchUpTest { + + @Test + public void testReconnectCatchUpRebuildsDictionary() throws Exception { + // Connection 1: send "alpha" (id 0), ACK it, then drop the socket so the + // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1). + // Without catch-up, connection 2's first data frame would carry + // deltaStart=1 and the fresh server would never learn id 0. + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + + // Let the I/O loop observe the server-side close before the next + // batch, so batch 2 is what drives the reconnect + catch-up. + Thread.sleep(200); + + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // The fresh (2nd) connection's dictionary, rebuilt purely from the + // frames it received, must hold both symbols contiguously with no + // null gap -- exactly what the catch-up frame guarantees. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals("alpha", conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + } + + @Test + public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception { + // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes), + // then drops once the server has learned them all. On reconnect the fresh + // server has an empty dictionary, so the I/O thread must replay all 40 as a + // catch-up -- but the server advertises a 160-byte batch cap, so the whole + // dictionary cannot fit in a single frame. The catch-up therefore splits + // into several contiguous zero-table frames that the fresh server stitches + // back into a complete, gap-free dictionary. + final int symbolCount = 40; + SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // One row per flush so each frame stays under the 160-byte cap; the + // sent dictionary still accumulates all 40 symbols on connection 1. + for (int i = 0; i < symbolCount; i++) { + sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + sender.flush(); + } + // Wait until connection 1 has learned every symbol, so the sender's + // sent-dictionary mirror (the catch-up source) holds all of them. + waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); + + // Let the I/O loop observe the server-side close before the next + // batch, so batch 2 drives the reconnect + split catch-up. + Thread.sleep(200); + + sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= symbolCount + 1, 10_000); + } + + // Connection 2's dictionary, rebuilt purely from the frames it received, + // must hold every symbol contiguously with no null gap -- the split + // catch-up frames reassemble exactly. + List conn2 = handler.dictFor(2); + Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); + for (int i = 0; i <= symbolCount; i++) { + Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + } + // The catch-up had to span more than one zero-table frame to stay under + // the advertised cap -- that split is the behaviour under test. + Assert.assertTrue("catch-up split into multiple frames (saw " + + handler.zeroTableFramesOnConn2 + ")", + handler.zeroTableFramesOnConn2 >= 2); + } + } + + private static String symbolName(int i) { + // 10-char symbols so 40 of them clearly exceed the advertised 160-byte cap. + return "symbol" + (1000 + i); + } + + private static int readVarint(byte[] buf, int[] pos) { + int result = 0; + int shift = 0; + while (pos[0] < buf.length) { + int b = buf[pos[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) return result; + shift += 7; + if (shift > 28) throw new IllegalStateException("varint too long"); + } + throw new IllegalStateException("varint truncated"); + } + + private static void waitFor(BoolCondition cond, long timeoutMillis) { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (cond.test()) return; + try { + Thread.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + Assert.fail("interrupted"); + } + } + Assert.fail("waitFor timed out"); + } + + @FunctionalInterface + private interface BoolCondition { + boolean test(); + } + + /** + * Mirrors the server's per-connection delta-dictionary accumulation: the + * dictionary resets on every new connection, and each frame's delta section + * ({@code setQuick(deltaStart + i)}, null-padding to reach deltaStart) extends + * or overwrites it. A missing catch-up would show up here as a null gap. + */ + private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + volatile boolean sawZeroTableFrameOnConn2; + private final List> dictsByConn = new CopyOnWriteArrayList<>(); + private TestWebSocketServer.ClientHandler currentClient; + private final AtomicLong nextSeq = new AtomicLong(0); + + synchronized List dictFor(int connNumber) { + return connNumber <= dictsByConn.size() + ? dictsByConn.get(connNumber - 1) + : new ArrayList<>(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + boolean newConnection = currentClient != client; + if (newConnection) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + } + int connNumber = dictsByConn.size(); + List dict = dictsByConn.get(connNumber - 1); + accumulate(data, dict); + if (connNumber == 2 && tableCount(data) == 0) { + sawZeroTableFrameOnConn2 = true; + } + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + // Drop the first connection right after ACKing its only frame, + // forcing the sender to reconnect onto a fresh dictionary. + if (connNumber == 1) { + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private static void accumulate(byte[] frame, List dict) { + final byte FLAG_DELTA_SYMBOL_DICT = 0x08; + if (frame.length < 12 || (frame[5] & FLAG_DELTA_SYMBOL_DICT) == 0) { + return; + } + int[] pos = {12}; // just past the 12-byte QWP header + int deltaStart = readVarint(frame, pos); + int deltaCount = readVarint(frame, pos); + while (dict.size() < deltaStart) { + dict.add(null); // null-pad, mirroring the server + } + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(frame, pos); + String symbol = new String(frame, pos[0], len, StandardCharsets.UTF_8); + pos[0] += len; + int idx = deltaStart + i; + while (dict.size() <= idx) { + dict.add(null); + } + dict.set(idx, symbol); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + + private static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + } + + /** + * Like {@link CatchUpHandler}, but drops connection 1 only after it has + * learned the whole batch, and counts the zero-table catch-up frames on + * connection 2 so a test can assert the dictionary replay split across + * several frames to respect the advertised batch cap. + */ + private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + volatile int zeroTableFramesOnConn2; + private final List> dictsByConn = new CopyOnWriteArrayList<>(); + private final int dropConn1AtDictSize; + private final AtomicLong nextSeq = new AtomicLong(0); + private boolean conn1Dropped; + private TestWebSocketServer.ClientHandler currentClient; + + SplitCatchUpHandler(int dropConn1AtDictSize) { + this.dropConn1AtDictSize = dropConn1AtDictSize; + } + + synchronized List dictFor(int connNumber) { + return connNumber <= dictsByConn.size() + ? dictsByConn.get(connNumber - 1) + : new ArrayList<>(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + boolean newConnection = currentClient != client; + if (newConnection) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + } + int connNumber = dictsByConn.size(); + List dict = dictsByConn.get(connNumber - 1); + CatchUpHandler.accumulate(data, dict); + if (connNumber == 2 && CatchUpHandler.tableCount(data) == 0) { + zeroTableFramesOnConn2++; + } + try { + client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + // Drop connection 1 only once it has learned the entire batch, so + // the sender's sent-dictionary mirror is complete and the reconnect + // catch-up must replay a dictionary larger than the batch cap. + if (connNumber == 1 && !conn1Dropped && dict.size() >= dropConn1AtDictSize) { + conn1Dropped = true; + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java new file mode 100644 index 00000000..88afc3ef --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -0,0 +1,352 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.Files; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * End-to-end recovery for the delta symbol dictionary in store-and-forward mode. + *

+ * A file-mode sender writes delta-encoded SYMBOL frames (each frame carries only + * the ids it introduces) to a slot but never drains it -- simulating a crash. A + * fresh sender then recovers the slot and replays those non-self-sufficient + * frames to a brand-new server whose dictionary starts empty. Correctness hinges + * on the persisted {@code .symbol-dict}: the recovering sender loads it, the I/O + * thread re-registers the whole dictionary via a catch-up frame, and only then do + * the delta frames replay. This test reconstructs the server-side dictionary from + * the wire and asserts it comes out complete and gap-free. + */ +public class DeltaDictRecoveryTest { + + private static final int DISTINCT_SYMBOLS = 8; + private static final int ROWS = 40; + private String sfDir; + + @Before + public void setUp() { + sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-delta-recov-" + System.nanoTime()).toString(); + } + + @After + public void tearDown() { + if (sfDir != null) { + rmDirRec(sfDir); + } + } + + @Test + public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception { + // Phase 1: silent server (no acks). Sender 1 writes symbol rows and + // close-fast (no drain), leaving unacked delta frames + a persisted + // dictionary in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String pad = TestUtils.repeat("x", 64); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .stringColumn("p", pad) + .longColumn("v", i) + .atNow(); + s1.flush(); + } + } + } + + // Phase 2: fresh server that reconstructs its per-connection dictionary + // from the delta sections. Sender 2 recovers the slot and replays. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } + } + + // The recovering sender must have re-registered the dictionary via a + // catch-up (0-table) frame before replaying delta frames. + Assert.assertTrue("recovery sent a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + // The reconstructed dictionary must be complete and gap-free: exactly + // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. + List dict = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + } + } + } + + @Test + public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { + // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. + // Silent server + close-fast leaves all frames unacked in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Simulate a host/power crash: the segment frames survive but the persisted + // dictionary is lost, and the ack watermark was left mid-stream. Truncate + // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at + // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The replay guard must + // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + Thread.sleep(1_000); // let the I/O loop attempt replay and hit the guard + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + terminal = e; + } + } + Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", + 0, handler.frames.get()); + Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); + } + } + + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { + byte[] buf = new byte[16]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.putInt(0x31574B41); // 'AKW1' + bb.putInt(0); // reserved + bb.putLong(fsn); + java.nio.file.Files.write(path, buf); + } + + private static int readVarint(byte[] buf, int[] pos) { + int result = 0; + int shift = 0; + while (pos[0] < buf.length) { + int b = buf[pos[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + if (shift > 28) { + throw new IllegalStateException("varint too long"); + } + } + throw new IllegalStateException("varint truncated"); + } + + private static void rmDirRec(String dir) { + if (!Files.exists(dir)) { + return; + } + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRec(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + /** + * Reconstructs the per-connection symbol dictionary from delta sections, + * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding. + */ + private static class DictReconstructingHandler implements TestWebSocketServer.WebSocketServerHandler { + volatile boolean sawCatchUpFrame; + private final List dict = new ArrayList<>(); + private final AtomicLong nextSeq = new AtomicLong(0); + private TestWebSocketServer.ClientHandler currentClient; + + synchronized List dictSnapshot() { + return new ArrayList<>(dict); + } + + synchronized int maxDictSize() { + return dict.size(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + dict.clear(); // fresh server dictionary per connection + } + accumulate(data); + if (tableCount(data) == 0 && hasDelta(data)) { + sawCatchUpFrame = true; + } + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + + private static boolean hasDelta(byte[] frame) { + return frame.length >= 12 && (frame[5] & 0x08) != 0; + } + + private static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + + private void accumulate(byte[] frame) { + if (!hasDelta(frame)) { + return; + } + int[] pos = {12}; + int deltaStart = readVarint(frame, pos); + int deltaCount = readVarint(frame, pos); + while (dict.size() < deltaStart) { + dict.add(null); + } + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(frame, pos); + String sym = new String(frame, pos[0], len, StandardCharsets.UTF_8); + pos[0] += len; + int idx = deltaStart + i; + while (dict.size() <= idx) { + dict.add(null); + } + dict.set(idx, sym); + } + } + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- sender leaves everything unacked in the slot + } + } + + /** Counts every binary frame it receives and acks it. */ + private static class CountingHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger frames = new AtomicInteger(); + private final AtomicLong nextSeq = new AtomicLong(0); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + frames.incrementAndGet(); + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java index 36553388..7ba7a1fe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java @@ -52,9 +52,10 @@ * disconnect = sender broken, every subsequent batch threw. Reconnect * machinery now handles transient drops: detect, build a fresh client * via the registered factory, reset wire state, and reposition the replay - * cursor at {@code engine.ackedFsn() + 1}. Cursor frames are self-sufficient - * (every frame carries full schema + full symbol-dict delta), so post-reconnect - * replay needs no producer-side schema-reset signal. + * cursor at {@code engine.ackedFsn() + 1}. Every frame carries its full schema + * inline; the symbol dictionary is either shipped in full per frame (file-mode + * store-and-forward) or re-registered by an I/O-thread catch-up frame before + * replay (memory-mode), so post-reconnect replay needs no producer-side reset. *

* This commit covers the mechanics with a single-attempt retry; backoff, * per-outage time cap, and auth-failure detection follow. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 45275528..b0a1b5a7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -32,23 +32,26 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; /** - * Pins down the "every frame on disk is self-sufficient" rule for the symbol - * dictionary. + * Pins down how the symbol dictionary is framed on the wire. *

- * The cursor SF path used to elide previously-sent symbols on subsequent - * batches over the same connection, emitting a delta-dict that carried only - * the new entries. That's wrong for SF: the bytes survive process restarts and - * replay against fresh server connections (post-reconnect, or via a background - * drainer adopting an orphan slot). A delta that references symbol ids the new - * server has never seen is unrecoverable. + * Both engine modes ship monotonic deltas -- each symbol id travels once, + * not the whole dictionary per message -- which is the bandwidth win this feature + * adds. The I/O thread re-registers the dictionary with a catch-up frame whenever + * it (re)connects, so a fresh server can resolve the non-self-sufficient delta + * frames that follow. *

- * Today every frame must carry a complete symbol-dict delta starting at id 0 - * (column schemas travel inline on the first batch too). This test asserts the - * symbol-dict invariant on the wire. + * The modes differ only in where the catch-up's dictionary comes from: memory + * mode keeps it in an in-process mirror; file-backed store-and-forward keeps it in + * a per-slot {@code .symbol-dict} file so a recovered or orphan-drained slot (a + * fresh process with no in-memory mirror) can rebuild it. This test asserts the + * monotonic wire framing in both modes and the presence of that dictionary file. */ public class SelfSufficientFramesTest { @@ -56,12 +59,72 @@ public class SelfSufficientFramesTest { private static final int DELTA_START_OFFSET = 12; @Test - public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception { - // Send two batches against the same connection, each with a - // distinct symbol value. With the old schema-ref/delta encoding, - // batch 2 would emit deltaStart=1, deltaCount=1 — only the new - // symbol. With self-sufficient frames, batch 2 must emit - // deltaStart=0 covering BOTH symbols. + public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { + // File-backed SF also ships monotonic deltas now: batch 2 carries only + // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict + // so a recovered/orphan-drained slot can rebuild it. + Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient"); + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // The engine places slot files under sf_dir/ (default "default"). + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check the persisted dictionary while the sender is live: a + // fully-drained close intentionally unlinks it (slot cleanup). + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); + } + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); + // batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } finally { + rmDir(sfDir); + } + } + + private static boolean containsUtf8(byte[] haystack, String needle) { + byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8); + outer: + for (int i = 0; i + n.length <= haystack.length; i++) { + for (int j = 0; j < n.length; j++) { + if (haystack[i + j] != n[j]) { + continue outer; + } + } + return true; + } + return false; + } + + @Test + public void testMemoryModeShipsMonotonicDelta() throws Exception { + // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries + // only "beta" as a delta starting at id 1, not the whole dictionary. CapturingHandler handler = new CapturingHandler(); try (TestWebSocketServer server = new TestWebSocketServer(handler)) { int port = server.getPort(); @@ -82,29 +145,17 @@ public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception { byte[] b1 = handler.batches.get(0); byte[] b2 = handler.batches.get(1); - // The deltaStart varint sits right after the 12-byte header. - // For self-sufficient frames it must be 0 (single byte 0x00) - // in BOTH batches — regardless of how many symbols the prior - // batch already shipped. - int deltaStart1 = readVarint(b1, DELTA_START_OFFSET); - int deltaStart2 = readVarint(b2, DELTA_START_OFFSET); - Assert.assertEquals("batch 1 deltaStart must be 0", 0, deltaStart1); - Assert.assertEquals("batch 2 deltaStart must be 0 (self-sufficient)", - 0, deltaStart2); - - // batch 2 must include >= 2 symbols in its delta dict (alpha - // from the prior batch + beta from this one). The varint at - // DELTA_START_OFFSET+1 is deltaCount. - int deltaCount2 = readVarint(b2, DELTA_START_OFFSET + 1); - Assert.assertTrue("batch 2 must redefine at least 2 symbols, got " + deltaCount2, - deltaCount2 >= 2); - - // Sanity: batch 2 should NOT be much smaller than batch 1 — - // with schema-ref/delta encoding it would have been; with - // self-sufficient frames the size is in the same ballpark. - Assert.assertTrue("batch 2 (" + b2.length + " bytes) must not be drastically smaller than batch 1 (" - + b1.length + ")", - b2.length >= b1.length / 2); + // Batch 1 introduces alpha at id 0. + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + + // Batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); } } @@ -122,6 +173,25 @@ private static int readVarint(byte[] buf, int offset) { throw new IllegalStateException("varint truncated"); } + private static void rmDir(Path dir) { + try { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } catch (IOException ignored) { + // best-effort + } + } + private static void waitFor(BoolCondition cond, long timeoutMillis) { long deadline = System.currentTimeMillis() + timeoutMillis; while (System.currentTimeMillis() < deadline) { @@ -157,6 +227,7 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat } } + // Mirrors WebSocketResponse STATUS_OK layout: status u8 | sequence u64 | table_count u16 static byte[] buildAck(long seq) { byte[] buf = new byte[1 + 8 + 2]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java new file mode 100644 index 00000000..10148468 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -0,0 +1,219 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.std.ObjList; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class PersistedSymbolDictTest { + + @Test + public void testAppendPersistsAcrossReopen() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals(0, d.size()); + d.appendSymbol("AAPL"); + d.appendSymbol("GOOG"); + d.appendSymbol("MSFT"); + Assert.assertEquals(3, d.size()); + } finally { + d.close(); + } + + // Reopen: entries recovered in id order. + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(reopened); + try { + Assert.assertEquals(3, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(3, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertTrue(reopened.loadedEntriesLen() > 0); + + // Appending after recovery continues from the recovered tip. + reopened.appendSymbol("TSLA"); + Assert.assertEquals(4, reopened.size()); + } finally { + reopened.close(); + } + + PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(4, third.size()); + Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3)); + } finally { + third.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testBadMagicIsRecreatedEmpty() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // A file with the right size but garbage content (bad magic). + Path f = dir.resolve(".symbol-dict"); + Files.write(f, new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals("bad-magic file recreated empty", 0, d.size()); + d.appendSymbol("X"); + Assert.assertEquals(1, d.size()); + } finally { + d.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testEmptySymbolRoundTrips() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol(""); + d.appendSymbol("nonempty"); + } finally { + d.close(); + } + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("", s.getQuick(0)); + Assert.assertEquals("nonempty", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testRemoveOrphanDeletesFile() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("A"); + d.close(); + Path f = dir.resolve(".symbol-dict"); + Assert.assertTrue(Files.exists(f)); + PersistedSymbolDict.removeOrphan(dir.toString()); + Assert.assertFalse(Files.exists(f)); + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testTornTrailingEntrySelfHeals() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Write two complete entries, then a torn trailing record: a + // length prefix of 5 followed by only 2 bytes (crash mid-append). + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("one"); + d.appendSymbol("two"); + d.close(); + + Path f = dir.resolve(".symbol-dict"); + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, + StandardOpenOption.APPEND); + + // Reopen: the torn tail is ignored; only the two complete entries load. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + // The next append overwrites the torn tail, keeping the file consistent. + re.appendSymbol("three"); + Assert.assertEquals(3, re.size()); + } finally { + re.close(); + } + + PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(3, re2.size()); + Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2)); + } finally { + re2.close(); + } + } finally { + rmDir(dir); + } + }); + } + + private static void rmDir(Path dir) { + try { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + } + }); + } catch (IOException ignored) { + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 6d4c5ef0..5e009d67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -108,6 +108,11 @@ public class TestWebSocketServer implements Closeable { // 401, 403, 404, 426, 503, etc. that the failover loop should // classify per failover.md §6. private volatile String rejectingStatusReason; + // When > 0, 101 upgrade responses advertise this value as the + // X-QWP-Max-Batch-Size header, capping the QWP message size the client + // builds. Lets a test force the delta-dictionary catch-up to split across + // several frames. Live-updatable via setAdvertisedMaxBatchSize(). + private volatile int advertisedMaxBatchSize; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -221,6 +226,14 @@ public int liveConnectionCount() { return liveConnections.get(); } + /** + * Advertises {@code X-QWP-Max-Batch-Size: } on subsequent + * handshakes (live update). Pass {@code 0} to stop advertising a cap. + */ + public void setAdvertisedMaxBatchSize(int maxBatchSize) { + this.advertisedMaxBatchSize = maxBatchSize; + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -603,6 +616,10 @@ private boolean performHandshake() throws IOException { if (role != null) { sb.append("X-QuestDB-Role: ").append(role).append("\r\n"); } + int maxBatch = advertisedMaxBatchSize; + if (maxBatch > 0) { + sb.append("X-QWP-Max-Batch-Size: ").append(maxBatch).append("\r\n"); + } sb.append("\r\n"); out.write(sb.toString().getBytes(StandardCharsets.US_ASCII)); out.flush(); From 30922301ef77d53ab68dab3da1d8f4e9dc4db1f3 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:10:41 +0100 Subject: [PATCH 02/78] Contain QWP dict catch-up send failures in the reconnect loop The symbol-dictionary catch-up called fail() on a send error, but the catch-up runs inside connectLoop (via swapClient) and, on the initial connect, on the caller thread (via start() -> positionCursorForStart). Calling fail() there re-entered connectLoop. On a reconnect this corrupted the wire mapping: the outer setWireBaselineWithCatchUp overwrote fsnAtZero while nextWireSeq kept the nested attempt's value, so a later ACK translated through engine.acknowledge(fsnAtZero + wireSeq) and trimmed un-acked frames from the store-and-forward log -- silent data loss. A flapping connection recursed connectLoop until the stack overflowed into a terminal, turning a transient outage into a hard failure (breaking Invariant B). On the initial connect the same fail() ran connectLoop on the caller thread and blocked Sender construction forever. sendDictCatchUp and sendCatchUpChunk now throw CatchUpSendException instead of calling fail(). connectLoop's own retry catch handles the swapClient path (one non-re-entrant reconnect with backoff); trySendOne's orphan-retire re-anchor turns it into a fresh fail() from the I/O loop body; start() drops the dead client so the I/O thread reconnects and re-sends the catch-up off the caller thread. A single dictionary entry too large for the server batch cap is non-retriable, so it latches a terminal (recordFatal) rather than looping -- also removing the oversized-entry reconnect livelock. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 109 ++++++++++++++---- 1 file changed, 84 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index cbf948b0..25ebcadd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1030,7 +1030,28 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // A recovered sender re-registers its dictionary with a catch-up on + // the very first connect. Here that runs on the CALLER thread (sync + // start), so we must NOT let it drive connectLoop -- that would block + // Sender construction forever on a transient outage. Drop the dead + // client instead: the I/O thread then reconnects via + // attemptInitialConnect -> swapClient and re-sends the catch-up off + // this thread. If the failure was already terminal (recordFatal set + // running=false, e.g. an entry too large for the batch cap), the I/O + // thread simply winds down and checkError() surfaces it. + WebSocketClient dead = client; + client = null; + if (dead != null) { + try { + dead.close(); + } catch (Throwable ignored) { + // best-effort + } + } + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { @@ -2028,14 +2049,6 @@ private long readVarintAt(long p, long limit) { return value; } - /** - * Builds a table-less catch-up frame carrying the full symbol dictionary - * ({@code deltaStart=0, deltaCount=sentDictCount}) and sends it to the freshly - * connected server, then consumes wire seq 0 for it. {@code sendBinary} copies - * the bytes synchronously, so the temporary frame buffer is freed immediately. - * On send failure it fails the loop, which recycles and retries on the next - * connection. - */ /** * Re-registers the whole symbol dictionary on a fresh connection, split into * as many table-less frames as the server's advertised batch cap requires so @@ -2043,8 +2056,10 @@ private long readVarintAt(long p, long limit) { * Each chunk carries a contiguous id range {@code [start .. start+count)}, in * order, so the server accumulates them exactly as it would the original * per-frame deltas. Returns the number of frames sent (each consumed a wire - * sequence), so the caller can align {@code fsnAtZero}. Stops early and fails - * the loop on a send error or an entry too large for the cap. + * sequence), so the caller can align {@code fsnAtZero}. Throws {@link + * CatchUpSendException} on a send error (retriable -- the caller reconnects); + * a single entry too large for the cap is non-retriable, so it latches a + * terminal before throwing. */ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); @@ -2065,15 +2080,19 @@ private int sendDictCatchUp() { long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] long entryBytes = entryEnd - entryStart; if (entryBytes > budget) { - fail(new LineSenderException( + // Non-retriable: the entry will not shrink and the same cluster + // re-advertises the same cap, so reconnecting would livelock. + // Latch a terminal (the data must be resent after the cap is + // raised) rather than calling fail() -- which, from inside the + // catch-up, would re-enter connectLoop (see CatchUpSendException). + LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "entryBytes=" + entryBytes + ", budget=" + budget + ']')); - return framesSent; + + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); + recordFatal(err); + throw new CatchUpSendException(err); } if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { - if (!sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { - return framesSent; - } + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; chunkStartId += chunkSymbols; chunkStartAddr = entryStart; @@ -2084,7 +2103,8 @@ private int sendDictCatchUp() { chunkBytes += entryBytes; p = entryEnd; } - if (chunkSymbols > 0 && sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + if (chunkSymbols > 0) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; } return framesSent; @@ -2092,10 +2112,12 @@ private int sendDictCatchUp() { /** * Sends one table-less catch-up frame carrying dictionary ids - * {@code [deltaStart .. deltaStart+deltaCount)}. Returns false and fails the - * loop on a send error. + * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link + * CatchUpSendException} on a send error instead of calling {@link #fail} + * (see that type for why the catch-up must not re-enter the reconnect loop); + * the caller turns it into a single, non-re-entrant reconnect. */ - private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { int payloadLen = NativeBufferWriter.varintSize(deltaStart) + NativeBufferWriter.varintSize(deltaCount) + symbolsLen; @@ -2116,15 +2138,20 @@ private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAdd Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); client.sendBinary(frame, frameLen); } catch (Throwable t) { - fail(t); - return false; + // Do NOT fail() here -- see CatchUpSendException. Signal the failure + // up so exactly one non-re-entrant reconnect follows. A JVM Error is + // never a transient reconnect case; let it propagate as-is so the + // I/O loop latches it terminal rather than looping on it. + if (t instanceof Error) { + throw (Error) t; + } + throw new CatchUpSendException(t); } finally { Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); } nextWireSeq++; // this catch-up chunk consumed a wire sequence lastFrameOrPingNanos = System.nanoTime(); totalFramesSent.incrementAndGet(); - return true; } private static long writeVarintAt(long addr, long value) { @@ -2169,7 +2196,15 @@ private boolean trySendOne() { // Nothing sent on this connection yet: re-anchor in place past // the retired tail. The wireSeq<->FSN mapping is untouched // because no wire sequence has been consumed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // Re-anchor's catch-up send failed. fail() here is a fresh, + // non-re-entrant connectLoop entry from the I/O loop body -- + // the same recovery a normal trySendOne send failure takes. + fail(e.getCause()); + return false; + } return true; } // Frames were already sent on this connection: the linear @@ -2336,6 +2371,30 @@ public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; } + /** + * Signals that a symbol-dictionary catch-up frame could not be sent on the + * current connection. Thrown by {@link #sendDictCatchUp}/{@link + * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs + * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial + * connect, inside {@link #start()} / {@link #trySendOne} on the caller + * thread. Calling {@code fail()} from there would re-enter {@code + * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire + * mapping (a subsequent ACK then trims un-acked frames) and growing the + * stack until it overflows into a terminal, breaking the "retry a transient + * outage forever" invariant -- or run {@code connectLoop} on the caller + * thread and block {@code Sender} construction indefinitely. Each catch + * site instead turns it into ONE non-re-entrant reconnect: {@code + * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} + * from the I/O loop body (trySendOne path), or dropping the dead client so + * the I/O thread reconnects (start path). A JVM {@code Error} is never + * wrapped -- it must stay terminal. + */ + private static final class CatchUpSendException extends RuntimeException { + CatchUpSendException(Throwable cause) { + super(cause); + } + } + /** * One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table * (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0) From 9f3f7bdc8b8f1109781d6f92196d283068d15eb9 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:24:13 +0100 Subject: [PATCH 03/78] Resume the QWP symbol-dict write-ahead from the durable size persistNewSymbolsBeforePublish keyed the append range off sentMaxSymbolId+1. That watermark only advances after the whole frame is published, whereas PersistedSymbolDict.size() advances per persisted entry. If a mid-batch appendSymbol threw (a short write on a full disk), the symbols before the failing one were already durable but the frame was not published, so sentMaxSymbolId stayed put. A retry then re-keyed from sentMaxSymbolId+1 and re-appended that already-persisted prefix, duplicating entries and breaking the dense id->symbol mapping recovery relies on (entry i must be symbol id i) -- a torn dictionary that re-registers the wrong symbols on the fresh server, or diverges the producer's watermark from the I/O thread's mirror. Resume from pd.size() instead: it is exactly the count already durable, so the retry continues past the persisted prefix (the next append overwrites any torn trailing bytes) without duplicating. In the happy path pd.size() equals sentMaxSymbolId+1, so behaviour is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 286596c4..216794ff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3665,12 +3665,22 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - int from = sentMaxSymbolId + 1; + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1. + // appendSymbol advances pd.size() per persisted entry, whereas + // sentMaxSymbolId only advances after the WHOLE frame is published (via + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior + // appendSymbol threw mid-batch (a short write -- disk full/quota), the + // frame was not published and sentMaxSymbolId stayed put, while the + // symbols before the failing one are already on disk. Keying the resume + // point off sentMaxSymbolId+1 would then re-append that persisted prefix + // on the retry, duplicating entries and corrupting the dense id->symbol + // mapping recovery relies on (position i must be symbol id i). pd.size() + // resumes exactly past what is already durable -- the next appendSymbol + // overwrites any torn trailing bytes at appendOffset -- so the + // write-ahead is idempotent under retry. In the happy path pd.size() + // equals sentMaxSymbolId+1, so behaviour is unchanged. int to = currentBatchMaxSymbolId; - if (to < from) { - return; - } - for (int id = from; id <= to; id++) { + for (int id = pd.size(); id <= to; id++) { pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); } } From 09cd706af158f97ecb1299bd14024f52a5a6cb56 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:24:23 +0100 Subject: [PATCH 04/78] Guard the oversized catch-up entry terminal Add a regression test: a dictionary entry larger than the reconnect server's per-chunk catch-up budget must latch a clean terminal, not reconnect-loop. Connection 1 advertises no cap so a ~200-byte symbol registers into the sent-dictionary mirror; the handler then shrinks the advertised cap and drops the socket, so the reconnect's catch-up cannot re-ship the entry. The test asserts the surfaced terminal names the catch-up path ("... during catch-up"). Reverting the fix (entry-too-large calling fail() again) fails this test with a StackOverflowError on the I/O thread -- the catch-up re-entering connectLoop -- confirming the guard bites both ways. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictCatchUpTest.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 5e9af978..1af55bc3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -25,7 +25,9 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -92,6 +94,65 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { } } + @Test + public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { + // A dictionary entry that exceeds the reconnect server's per-chunk budget + // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. + // sendDictCatchUp must latch a clean terminal ("... during catch-up") + // rather than call fail(): pre-fix the oversized entry drove an endless + // reconnect loop (the entry never shrinks and the same cluster + // re-advertises the same cap) and re-entered connectLoop from the catch-up. + // + // Connection 1 advertises no cap, so the ~202-byte symbol registers and + // enters the sent-dictionary mirror. The handler then shrinks the + // advertised cap to 160 (catch-up budget 132) and drops the socket, so the + // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do + // this: the client refuses to SEND a single-table frame over the cap, and + // that data frame is always larger than the bare catch-up entry.) + CapShrinkHandler handler = new CapShrinkHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + handler.setServer(server); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry + LineSenderException terminal = null; + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + try { + sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); + sender.flush(); + // The terminal latches on the I/O thread once the reconnect's + // catch-up hits the oversized entry; it surfaces to the producer + // on a subsequent flush. Poll a bounded time for it. The polling + // rows use a small symbol that fits the shrunk cap, so the + // producer-side cap check never fires and flush() surfaces the + // I/O thread's catch-up terminal via checkError. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + sender.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); + Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up")); + } + } + @Test public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception { // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes), @@ -267,6 +328,47 @@ private static int tableCount(byte[] frame) { } } + /** + * ACKs connection 1's frames with no advertised cap (so an oversized symbol + * registers), then -- once connection 1 has sent something -- shrinks the + * advertised batch cap and drops the socket. The reconnect (connection 2) + * therefore advertises a cap whose catch-up budget is too small for the + * symbol, exercising the oversized-entry terminal in sendDictCatchUp. + */ + private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + private final AtomicLong nextSeq = new AtomicLong(0); + private TestWebSocketServer.ClientHandler currentClient; + private volatile TestWebSocketServer server; + + void setServer(TestWebSocketServer server) { + this.server = server; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + } + try { + client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + if (connectionsAccepted.get() == 1) { + // Connection 1 registered the big symbol. Shrink the cap so the + // reconnect's catch-up budget can't fit it, then drop to force + // the reconnect. Setting the cap before the close (not from the + // test thread after it) removes the set-vs-reconnect race. + server.setAdvertisedMaxBatchSize(160); // catch-up budget = 132 + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + /** * Like {@link CatchUpHandler}, but drops connection 1 only after it has * learned the whole batch, and counts the zero-table catch-up frames on From 18dd545381d0f5c8a0a5fb547a26447b1fd32362 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:25:36 +0100 Subject: [PATCH 05/78] Persist the QWP symbol-dict batch in a single write persistNewSymbolsBeforePublish appended each new symbol with its own PersistedSymbolDict.appendSymbol call, and each appendSymbol issues one positioned write. A high-cardinality batch -- one new symbol per row, which is exactly the store-and-forward workload delta encoding targets -- therefore stalled the producer thread with up to one pwrite syscall per row per flush. Add PersistedSymbolDict.appendSymbols(dict, from, to): it encodes the whole [from..to] entry region into scratch once and issues a single positioned write, so a flush that introduces N symbols costs one syscall instead of N. It keeps appendSymbol's durability and idempotency contract -- no fsync, and a short write throws without advancing size, so a retry keyed off size() re-encodes and overwrites at the same offset. PersistedSymbolDictTest.testAppendSymbolsBatchWritesDenseRange checks the batched write produces the same dense, id-ordered file (including an empty symbol mid-range), that an empty range is a no-op, and that a follow-on batch keyed off the recovered size continues without a gap or duplicate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 34 +++++------ .../client/sf/cursor/PersistedSymbolDict.java | 41 +++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 59 +++++++++++++++++++ 3 files changed, 117 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 216794ff..54be9ea0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3665,24 +3665,24 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1. - // appendSymbol advances pd.size() per persisted entry, whereas + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write. + // + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: + // appendSymbols advances pd.size() only after a full write, whereas // sentMaxSymbolId only advances after the WHOLE frame is published (via - // advanceSentMaxSymbolId, after activeBuffer.write). If a prior - // appendSymbol threw mid-batch (a short write -- disk full/quota), the - // frame was not published and sentMaxSymbolId stayed put, while the - // symbols before the failing one are already on disk. Keying the resume - // point off sentMaxSymbolId+1 would then re-append that persisted prefix - // on the retry, duplicating entries and corrupting the dense id->symbol - // mapping recovery relies on (position i must be symbol id i). pd.size() - // resumes exactly past what is already durable -- the next appendSymbol - // overwrites any torn trailing bytes at appendOffset -- so the - // write-ahead is idempotent under retry. In the happy path pd.size() - // equals sentMaxSymbolId+1, so behaviour is unchanged. - int to = currentBatchMaxSymbolId; - for (int id = pd.size(); id <= to; id++) { - pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); - } + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist + // threw (a short write -- disk full/quota), the frame was not published + // and sentMaxSymbolId stayed put, while the symbols before the failing + // one are already on disk. Keying the resume point off sentMaxSymbolId+1 + // would then re-append that persisted prefix on the retry, duplicating + // entries and corrupting the dense id->symbol mapping recovery relies on + // (position i must be symbol id i). pd.size() resumes exactly past what is + // already durable (the write overwrites any torn trailing bytes at + // appendOffset), so the write-ahead is idempotent under retry. In the + // happy path pd.size() equals sentMaxSymbolId+1, so the range -- and the + // behaviour -- are unchanged; the single positioned write just replaces + // the previous one-syscall-per-new-symbol loop. + pd.appendSymbols(globalSymbolDictionary, pd.size(), currentBatchMaxSymbolId); } private void resetSymbolDictStateForNewConnection() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 4c2674b9..92a0cc78 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -173,6 +174,46 @@ public void appendSymbol(CharSequence symbol) { size++; } + /** + * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes + * the whole {@code [len varint][utf8]...} region into scratch first, then + * issues one positioned write -- versus one {@code pwrite(2)} per symbol via + * {@link #appendSymbol}. That per-symbol syscall count is the hot-path cost + * on a high-cardinality batch (one new symbol per row), which is exactly the + * store-and-forward workload delta encoding targets. Callers pass the + * dictionary and the range so the ids resolve to their symbol strings. + *

+ * Same durability and idempotency contract as {@link #appendSymbol}: no + * fsync, and a short write throws WITHOUT advancing {@code size}/{@code + * appendOffset}, so a retry keyed off {@link #size()} re-encodes the same + * range and overwrites at the same offset. No-op when the range is empty or + * the dictionary is closed. + */ + public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { + if (closed || to < from) { + return; + } + int len = 0; + for (int id = from; id <= to; id++) { + CharSequence symbol = dict.getSymbol(id); + int utf8Len = Utf8s.utf8Bytes(symbol); + int recLen = varintSize(utf8Len) + utf8Len; + ensureScratch(len + recLen); + long p = writeVarint(scratchAddr + len, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + len += recLen; + } + long written = Files.write(fd, scratchAddr, len, appendOffset); + if (written != len) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + len + ", actual=" + written + ']'); + } + appendOffset += len; + size += to - from + 1; + } + @Override public void close() { if (closed) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 10148468..73549689 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.ObjList; import org.junit.Assert; @@ -89,6 +90,64 @@ public void testAppendPersistsAcrossReopen() throws Exception { }); } + @Test + public void testAppendSymbolsBatchWritesDenseRange() throws Exception { + // appendSymbols persists a whole id range in one write (the hot-path + // syscall reduction). It must produce the same dense, id-ordered file a + // per-symbol loop would, including an empty symbol mid-range, and a second + // batched call keyed off size() must continue densely (the resume-from- + // durable-size contract the producer relies on). + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + dict.getOrAddSymbol("TSLA"); // id 3 + + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbols(dict, 0, 3); // one write for all four ids + Assert.assertEquals(4, d.size()); + d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op + Assert.assertEquals(4, d.size()); + } finally { + d.close(); + } + + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(4, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(4, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertEquals("TSLA", symbols.getQuick(3)); + + // A follow-on batch keyed off the recovered size continues + // the dense sequence without a gap or duplicate. + dict.getOrAddSymbol("NVDA"); // id 4 + reopened.appendSymbols(dict, reopened.size(), 4); + Assert.assertEquals(5, reopened.size()); + } finally { + reopened.close(); + } + + PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(5, third.size()); + Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4)); + } finally { + third.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testBadMagicIsRecreatedEmpty() throws Exception { assertMemoryLeak(() -> { From 2a1a2d636b4fa31dc264d923f9f285b0189cf0f7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:25:49 +0100 Subject: [PATCH 06/78] Free the recovery-seeded dict mirror in close() On recovery / orphan-drain the CursorWebSocketSendLoop constructor seeds a native mirror (sentDictBytesAddr) from the slot's persisted dictionary so the first connection can re-register it. That mirror is freed only on ioLoop's exit path, so a loop that is constructed but never runs -- start() never called, or Thread.start() failing before the loop runs, or a close() racing an unstarted loop -- leaked it. close() already safety-nets the client for that same "loop never started" case; the mirror was missed. close() now frees the mirror when the loop never ran (ioThread was null on entry). It does NOT free it when the loop ran: ioLoop's exit owns the free there, and on the failed-stop path the thread may still be mid-send, so touching the mirror would race; a duplicate close observes a zero address and skips. CursorWebSocketSendLoopMirrorLeakTest populates a recoverable slot, then leak-checks constructing an engine + loop over it and closing WITHOUT start(). Reverting the free fails it with a 4096-byte NATIVE_DEFAULT leak. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 17 +++ ...CursorWebSocketSendLoopMirrorLeakTest.java | 141 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 25ebcadd..2e6b826c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -788,6 +788,12 @@ public synchronized void close() { // after — the latch await is only skipped when the loop never ran. running = false; Thread t = ioThread; + // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets + // freed on ioLoop's exit path. When t == null the loop never ran (start() + // was never called, or t.start() failed before committing ioThread), so + // that free never happens and a seeded (recovery / orphan-drain) mirror + // would leak. Capture that here; free it below, after client teardown. + boolean loopNeverRan = t == null; if (t != null) { LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. @@ -846,6 +852,17 @@ public synchronized void close() { } client = null; } + // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never + // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and + // on the failed-stop path (interrupted latch await, ioThread left set) the + // thread may still be mid-send, so touching the mirror here would race. + // A duplicate close observes sentDictBytesAddr == 0 and skips. + if (loopNeverRan && sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + } } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java new file mode 100644 index 00000000..19626ba4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -0,0 +1,141 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Guards the recovery-seeded symbol-dictionary mirror against leaking when the + * I/O loop is constructed but never run. + *

+ * On recovery / orphan-drain the {@link CursorWebSocketSendLoop} constructor + * seeds a native mirror ({@code sentDictBytesAddr}) from the slot's persisted + * dictionary so the first connection can re-register it. That mirror is freed on + * the I/O thread's exit path -- so if the loop is closed WITHOUT ever starting + * (start() never called, or Thread.start() failing before the loop runs), the + * free never happens. {@code close()} must free it in that case. + */ +public class CursorWebSocketSendLoopMirrorLeakTest { + + private static final int DISTINCT_SYMBOLS = 8; + private static final int ROWS = 40; + + @Test + public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { + Path sfDir = Files.createTempDirectory("qwp-mirror-leak"); + try { + // Populate a slot with delta frames + a non-empty .symbol-dict, then + // abandon it (silent server, close-fast) -- outside assertMemoryLeak, + // because a full Sender+server round trip is not net-zero on its own. + populateRecoverableSlot(sfDir); + + Path slot = sfDir.resolve("default"); + Assert.assertTrue("populate must leave a persisted dictionary", + Files.exists(slot.resolve(".symbol-dict"))); + + // Only the recovery construct + close is leak-checked: the engine + // recovers (loading the dict), the loop ctor seeds the mirror from it, + // and close() -- with NO start() -- must free every native allocation. + // Pre-fix the seeded mirror leaks here and this assertion fails. + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull("disk-mode engine must open a persisted dict", pd); + Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)", + pd.size() > 0 && pd.loadedEntriesLen() > 0); + + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + // Close without start(): the ctor-seeded mirror is this + // thread's to free, since the I/O loop never ran. + loop.close(); + } + }); + } finally { + rmDir(sfDir); + } + } + + private static void populateRecoverableSlot(Path sfDir) throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .longColumn("v", i) + .atNow(); + s1.flush(); + } + } + } + } + + private static void rmDir(Path dir) throws IOException { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- the sender leaves everything unacked in the slot + } + } +} From e130da922fd35e981be682ea9609dbfb774f746d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:42:03 +0100 Subject: [PATCH 07/78] Pin the recovery test on the catch-up frame testRecoveredSlotReplaysDeltaFramesAgainstFreshServer never acked in phase 1, so recovery replayed from the very first frame -- whose delta already starts at id 0. The replayed frames were thus self-sufficient from 0, and the reconstructed-dictionary assertions passed whether or not the seeded catch-up carried the right symbols (or any at all). Only the sawCatchUpFrame existence check was load-bearing. Stamp the ack watermark at FSN DISTINCT_SYMBOLS-1 between the phases so recovery replays from the first frame past the symbol-introducing cycle: a frame with deltaStart=DISTINCT_SYMBOLS carrying no new symbols. The early ids it references now exist only in the persisted dictionary, so the reconstructed dictionary is complete solely because the catch-up re-registered them. Verified both ways: with a catch-up that sends a table-less frame but no symbols, the pre-change test still passes (the head frames carry the dictionary) while the stamped test fails at "dictionary id 0 expected sym-0 but was null". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cutlass/qwp/client/DeltaDictRecoveryTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 88afc3ef..46c2a66b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -104,6 +104,20 @@ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Excep } } + // Ack a prefix so recovery does NOT replay from the self-sufficient head. + // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the + // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN + // DISTINCT_SYMBOLS onward -- frames whose delta starts at + // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle + // reuse existing ids). The early ids those frames reference then exist + // ONLY in the persisted dictionary, so the reconstructed dictionary below + // is complete solely because the catch-up frame re-registered them. That + // pins the content assertions to the catch-up: without it (or with a + // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 + // and the per-id checks would fail. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); + // Phase 2: fresh server that reconstructs its per-connection dictionary // from the delta sections. Sender 2 recovers the slot and replays. DictReconstructingHandler handler = new DictReconstructingHandler(); From 2b78b0e5e0dd80ad7170cc16cea2e731e75a7331 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:42:13 +0100 Subject: [PATCH 08/78] Cover the disk-mode full-dict fallback When a disk-mode slot's .symbol-dict cannot be opened, the engine reports delta encoding as unavailable and the sender must fall back to self-sufficient frames -- every batch re-ships the whole dictionary from id 0 -- because a recovered slot would have no dictionary to rebuild non-self-sufficient deltas from. Nothing exercised that path. Add a test that plants a directory where the dictionary file belongs, so openRW / openCleanRW fail and open() returns null. It then asserts both batches ship deltaStart=0 and that batch 2 re-ships the whole dictionary (deltaCount=2), rather than the monotonic delta (deltaStart=1, deltaCount=1) the enabled path emits. Verified it bites: forcing isDeltaDictEnabled() to stay true regresses batch 2 to deltaStart=1 and the test fails. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index b0a1b5a7..10bd1e6b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -107,6 +107,62 @@ public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { } } + @Test + public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Exception { + // When the per-slot .symbol-dict cannot be opened in disk mode, + // isDeltaDictEnabled() is false and the sender must fall back to + // self-sufficient frames: every batch re-ships the WHOLE dictionary from + // id 0. A recovered / orphan-drained slot then has no dictionary to + // rebuild deltas from, so a monotonic delta would dangle ids on the fresh + // server -- the full-dict frame is the safe degradation. Force the open + // failure by planting a DIRECTORY where the dictionary file belongs: + // openRW / openCleanRW on a directory fails, so open() returns null. + Path sfDir = Files.createTempDirectory("qwp-sf-fallback"); + Path dictPath = sfDir.resolve("default").resolve(".symbol-dict"); + Files.createDirectories(dictPath); // a directory, not a file + Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + // The planted directory is untouched -- the dictionary never opened, + // so delta encoding stayed disabled. + Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", + Files.isDirectory(dictPath)); + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Full-dict fallback: BOTH batches start at id 0, and batch 2 re-ships + // the WHOLE dictionary (alpha + beta), NOT a monotonic delta (which + // would be deltaStart=1, deltaCount=1 as in the test above). + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", + 0, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", + 2, readVarint(b2, DELTA_START_OFFSET + 1)); + } finally { + rmDir(sfDir); + } + } + private static boolean containsUtf8(byte[] haystack, String needle) { byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8); outer: From 282ac09fbc3ffe920435964780506816d02f9899 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:52:59 +0100 Subject: [PATCH 09/78] Truncate the torn tail when reopening the symbol dict openExisting parsed complete entries and set appendOffset past the last one, but left the file at its full length. A crash mid-append leaves a torn trailing record; if the next append after recovery is SHORTER than that torn tail, it overwrites only the tail's prefix and leaves residue beyond its own end. A later recovery can then mis-parse that residue as a ghost symbol, shifting every subsequent dense id -- so the "self-healing tail" guarantee was not actually airtight. open() now truncates the file to the end of the last complete entry (ftruncate) so nothing survives past appendOffset. Best-effort: a failed truncate falls back to the prior overwrite-from-appendOffset behaviour. testTornTrailingEntrySelfHeals now asserts the file returns to its clean length after the reopen; reverting the truncate fails it (19 vs 16 bytes). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/sf/cursor/PersistedSymbolDict.java | 14 ++++++++++++-- .../client/sf/cursor/PersistedSymbolDictTest.java | 9 ++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 92a0cc78..15179e89 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -328,8 +328,18 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { } Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; - // appendOffset lands just past the last complete entry, so the next - // append overwrites any torn trailing bytes. + // Physically drop any torn/stale trailing bytes (a crash mid-append) + // so a LATER, shorter append cannot leave residue past its own end + // that a future recovery mis-parses as a ghost symbol -- which would + // shift every subsequent dense id. Without this, appendOffset already + // lands just past the last complete entry, so the next append + // overwrites FROM there, but only truncation guarantees nothing + // survives BEYOND it. Best-effort: a failed truncate just forgoes the + // hardening and falls back to the overwrite-from-appendOffset behaviour. + long validLen = HEADER_SIZE + entriesLen; + if (validLen < len) { + Files.truncate(fd, validLen); + } return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 73549689..df6bab21 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -229,17 +229,24 @@ public void testTornTrailingEntrySelfHeals() throws Exception { d.close(); Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); // Reopen: the torn tail is ignored; only the two complete entries load. PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); try { + // open() physically truncates the torn tail: the file returns + // to its clean length, so a later SHORTER append can never + // leave residue past its end that a future recovery mis-parses + // as a ghost symbol. + Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f)); Assert.assertEquals(2, re.size()); ObjList s = re.readLoadedSymbols(); Assert.assertEquals("one", s.getQuick(0)); Assert.assertEquals("two", s.getQuick(1)); - // The next append overwrites the torn tail, keeping the file consistent. + // The next append continues from the truncated tail cleanly. re.appendSymbol("three"); Assert.assertEquals(3, re.size()); } finally { From 754eb3d642ce991755bef93dc833b0be114fa7a0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:53:09 +0100 Subject: [PATCH 10/78] Guard the sent-dict mirror against int overflow The I/O thread's lifetime-monotonic symbol-dictionary mirror is sized with int math: accumulateSentDict passed sentDictBytesLen + regionBytes (an int sum) to ensureSentDictCapacity, and the grow step doubled capacity*2, also int. On a pathological, very-high-cardinality connection the sum overflows negative -- so the capacity check passes and copyMemory scribbles past the buffer (silent heap corruption) -- and capacity*2 overflows negative near 1 GB, degrading the doubling to exact-fit reallocs. Reaching this needs ~200M+ distinct symbols on one connection, far past any real workload, but the failure mode is silent corruption. ensureSentDictCapacity now takes a long, the caller passes a long sum, and the method throws a LineSenderException above an int-addressable ceiling (Integer.MAX_VALUE - 8) instead of overflowing, growing in long math clamped to that ceiling. Defensive only -- not reachable at realistic symbol cardinality, so there is no scale test. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2e6b826c..3c7a0421 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -133,6 +133,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror + // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even + // this needs ~200M+ distinct symbols on a single connection, far past any real + // workload. The guard exists so that pathological growth fails loudly instead + // of overflowing the int capacity math into a heap-corrupting copyMemory. + private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ @@ -2035,19 +2041,32 @@ private void accumulateSentDict(long payloadAddr, int payloadLen) { } } int regionBytes = (int) (p - regionStart); - ensureSentDictCapacity(sentDictBytesLen + regionBytes); + // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on + // a very-high-cardinality lifetime connection; ensureSentDictCapacity then + // fails loudly rather than overflowing to a negative int (which would make + // the capacity check pass and copyMemory scribble past the buffer). + ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; sentDictCount += (int) deltaCount; } - private void ensureSentDictCapacity(int required) { + private void ensureSentDictCapacity(long required) { if (sentDictBytesCapacity >= required) { return; } - int newCap = Math.max(sentDictBytesCapacity * 2, Math.max(4096, required)); - sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); - sentDictBytesCapacity = newCap; + if (required > MAX_SENT_DICT_BYTES) { + throw new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + } + // Grow in long to avoid the capacity*2 int overflow (negative) that would + // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. + long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required)); + if (newCap > MAX_SENT_DICT_BYTES) { + newCap = MAX_SENT_DICT_BYTES; + } + sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, (int) newCap, MemoryTag.NATIVE_DEFAULT); + sentDictBytesCapacity = (int) newCap; } private long readVarintAt(long p, long limit) { From 4f44fb3b8d880e416044b41cf1ebde0e348503e7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:09:25 +0100 Subject: [PATCH 11/78] Parse the delta header once per frame on send trySendOne decoded a frame's delta header twice: the pre-send torn-dictionary guard called frameDeltaStart (magic/flags check + start-id varint), then post-send accumulateSentDict re-ran isDeltaFrame and re-read the start id before reading deltaCount. Both run on every delta frame on the I/O send path. Decode the start id once in the guard, hoist the frame address into a local, and pass the start id into accumulateSentDict, which now locates deltaCount just past the canonical start-id encoding (via NativeBufferWriter.varintSize) instead of re-parsing the header. The non-delta-frame case is carried by the same start id (-1), so the post- send mirror update runs exactly when it did before. Also move the accumulateSentDict javadoc onto accumulateSentDict: it had drifted above frameDeltaStart (which kept its own doc), leaving accumulateSentDict undocumented. The per-entry region walk (to size the mirror copy) remains; eliminating it needs a wire-level deltaBytes field, a server-side change out of scope for this client fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3c7a0421..9f41e1e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1978,17 +1978,6 @@ private void setWireBaselineWithCatchUp(long replayStart) { } } - /** - * Copies the symbol-dictionary delta a just-sent frame carries into the - * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can - * re-register it. Frames are sent in FSN order carrying monotonically - * extending deltas, so a frame whose delta starts exactly at - * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame - * (nothing new) is skipped. Only ever called in delta mode. - * - * @param payloadAddr address of the QWP message (12-byte header first) - * @param payloadLen message length in bytes - */ /** * Returns the symbol-dictionary delta start id of a frame, or -1 when the * frame carries no delta section. Used by the pre-send torn-dictionary guard. @@ -2013,14 +2002,27 @@ private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; } - private void accumulateSentDict(long payloadAddr, int payloadLen) { + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode, for a frame the + * pre-send guard already classified as a delta frame. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + * @param deltaStart the frame's delta start id, already decoded by the + * pre-send guard ({@link #frameDeltaStart}) -- passed in so + * the magic/flags and start-id varint are not re-parsed + */ + private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) { long limit = payloadAddr + payloadLen; - if (!isDeltaFrame(payloadAddr, payloadLen)) { - return; - } - long p = payloadAddr + QwpConstants.HEADER_SIZE; - long deltaStart = readVarintAt(p, limit); - p = varintEnd; + // deltaStart is known (the guard decoded it); locate deltaCount just past + // its canonical LEB128 encoding rather than re-reading the header and the + // start-id varint. + long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); long deltaCount = readVarintAt(p, limit); p = varintEnd; // Only a delta that extends exactly from our current tip introduces new @@ -2291,6 +2293,11 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // -1 unless this is a delta frame; the guard decodes it once here and + // accumulateSentDict reuses it post-send, so the delta header is parsed + // once per frame rather than twice. + int deltaStart = -1; if (deltaDictEnabled) { // Torn-dictionary guard. In normal operation a delta frame's start id // never exceeds the dictionary coverage established so far (replayed @@ -2302,7 +2309,7 @@ private boolean trySendOne() { // Sending the frame would corrupt the table (the server would null-pad // the missing ids), so fail terminally instead; the unreplayable data // must be resent. - int deltaStart = frameDeltaStart(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + deltaStart = frameDeltaStart(frameAddr, payloadLen); if (deltaStart > sentDictCount) { recordFatal(new LineSenderException( "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " @@ -2312,16 +2319,16 @@ private boolean trySendOne() { } } try { - client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + client.sendBinary(frameAddr, payloadLen); } catch (Throwable t) { fail(t); return false; } - if (deltaDictEnabled) { + if (deltaStart >= 0) { // Mirror the symbols this frame introduced so a later reconnect can // rebuild the whole dictionary. Idempotent on replay: a frame whose // delta we already hold advances nothing. - accumulateSentDict(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + accumulateSentDict(frameAddr, payloadLen, deltaStart); } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; From 0f9d88f1c31befe03c4702cd08965585f6b96724 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:09:35 +0100 Subject: [PATCH 12/78] Fix stale self-sufficient-frame comments Several comments predated file-mode delta encoding and claimed every cursor frame is self-sufficient with a "symbol-dict delta from id 0". That is now only the fallback: in delta mode (memory mode, and file mode when the persisted dictionary opened) frames carry monotonic deltas that are NOT self-sufficient, and the fresh server's dictionary is re-established by an I/O-thread catch-up frame before replay. The worst offender was the deltaDictEnabled field doc ("Enabled only in memory-mode ... File-mode keeps full self-sufficient frames"), which directly contradicted the feature. Corrected it plus the two ensureConnected call-site comments, the append-failed-path comment, and the wasRecoveredFromDisk field doc (schema stays self-sufficient per frame; the dictionary does not). No behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 31 ++++++++++--------- .../client/sf/cursor/CursorSendEngine.java | 6 ++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 54be9ea0..77e85392 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -224,11 +224,14 @@ public class QwpWebSocketSender implements Sender { private boolean deferCommit; // True when the sender emits incremental (delta) symbol dictionaries: each // message carries only symbol ids not yet sent on the wire, rather than the - // full dictionary from id 0. Enabled only in memory-mode, where a reconnect - // replays from the in-process ring and the I/O thread re-registers the whole - // dictionary via a catch-up frame before replaying. File-mode store-and-forward - // keeps full self-sufficient frames (recovery/orphan-drain can replay mid-stream - // to a fresh server, where a partial delta would leave gaps). Set in setCursorEngine. + // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from + // the in-process ring) and in file-mode store-and-forward when the per-slot + // persisted dictionary opened. In both, the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying, so a non-self-sufficient + // delta frame never dangles an id on a fresh server. Falls back to full + // self-sufficient frames only when the persisted dictionary is unavailable in + // file-mode (recovery/orphan-drain would then have nothing to rebuild the + // deltas from). Set in setCursorEngine. private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after @@ -3386,18 +3389,18 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). The symbol-dict watermark still gets - // reset for consistency with the sync path; the post-connect replay - // path does not need a producer-side reset signal because every - // cursor frame is self-sufficient. + // its default version (V1). The per-batch symbol-dict watermark still + // gets reset for consistency with the sync path; the post-connect + // replay path needs no producer-side reset signal (see below). Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection, so reset the symbol-dict - // watermark. Cursor frames are self-sufficient (every frame carries its - // full inline schema + a symbol-dict delta from id 0), so post-reconnect - // replay needs no producer-side reset signal. + // Server starts fresh on each connection, so reset the per-batch + // symbol-dict watermark. Every frame still carries its full inline schema, + // and the fresh server's dictionary is re-established either by a full-dict + // frame (full-dict mode) or by an I/O-thread catch-up frame before replay + // (delta mode), so post-reconnect replay needs no producer-side reset signal. resetSymbolDictStateForNewConnection(); connectionError.set(null); @@ -3784,7 +3787,7 @@ private void sealAndSwapBuffer() { // back to it; flushPendingRows aborts its post-enqueue state // updates after this throw, so the source rows stay intact and the // next batch re-emits the same rows along with the full inline - // schema and symbol-dict delta from id 0. + // schema and the symbol-dict delta the batch requires. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 85015a73..3bc0a185 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -84,9 +84,9 @@ public final class CursorSendEngine implements QuietCloseable { private final SlotLock slotLock; // True when the constructor recovered an existing on-disk slot rather // than starting fresh. Diagnostic accessor for tests and observability; - // cursor frames are self-sufficient (every frame carries full schema + - // full symbol-dict delta), so producer-side schema reset on recovery - // is not required. + // every frame carries its full inline schema, so producer-side schema reset + // on recovery is not required (the symbol dictionary, which delta frames do + // NOT carry in full, is re-registered by an I/O-thread catch-up instead). private final boolean wasRecoveredFromDisk; // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a // ring recovered from disk, or -1 for fresh/memory rings and recovered From d89f14f6186e4ca8d96ce1cf250836af3fe7406d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:35:19 +0100 Subject: [PATCH 13/78] Make the delta-dict tests deterministic and leak-checked Two robustness fixes to the delta symbol-dictionary tests. Deterministic synchronization (replaces fixed sleeps): - DeltaDictCatchUpTest waited a fixed 200 ms for the server to close connection 1 before sending batch 2. On a loaded machine that could under-wait and let batch 2 race into connection 1's pre-close window, changing which connection the catch-up lands on. The handler now sets a conn1Closed flag after it closes the socket, and the test waits on that. - DeltaDictRecoveryTest's torn-dictionary test slept a fixed 1 s to let the replay guard fire before close(). It now polls flush() for the latched terminal (close() remains the fallback), so it captures the terminal as soon as it fires -- the run dropped from ~1 s to ~0.3 s. Leak checks: the Sender-based tests allocate native memory (the send-loop mirror, persisted-dict buffers, segment mmaps) but were not wrapped in assertMemoryLeak, unlike the rest of the suite. Wrap all eight methods across the three classes; every one is balanced (they already cleaned up via try-with-resources -- the wrapper now guards against future leaks). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictCatchUpTest.java | 227 +++++++++-------- .../qwp/client/DeltaDictRecoveryTest.java | 238 ++++++++++-------- .../qwp/client/SelfSufficientFramesTest.java | 220 ++++++++-------- 3 files changed, 367 insertions(+), 318 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 1af55bc3..0d2ff9f7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -42,6 +42,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode). *

@@ -61,37 +63,40 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1). // Without catch-up, connection 2's first data frame would carry // deltaStart=1 and the fresh server would never learn id 0. - CatchUpHandler handler = new CatchUpHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); - - // Let the I/O loop observe the server-side close before the next - // batch, so batch 2 is what drives the reconnect + catch-up. - Thread.sleep(200); - - sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.connectionsAccepted.get() >= 2 - && handler.dictFor(2).size() >= 2, 5_000); - } + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); - // The fresh (2nd) connection's dictionary, rebuilt purely from the - // frames it received, must hold both symbols contiguously with no - // null gap -- exactly what the catch-up frame guarantees. - List conn2 = handler.dictFor(2); - Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", - handler.sawZeroTableFrameOnConn2); - Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); - Assert.assertEquals("alpha", conn2.get(0)); - Assert.assertEquals("beta", conn2.get(1)); - } + // Wait until the server has actually closed connection 1 before + // sending batch 2, so batch 2 cannot race into connection 1 and + // must drive the reconnect + catch-up. + waitFor(() -> handler.conn1Closed, 5_000); + + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // The fresh (2nd) connection's dictionary, rebuilt purely from the + // frames it received, must hold both symbols contiguously with no + // null gap -- exactly what the catch-up frame guarantees. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals("alpha", conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + }); } @Test @@ -109,48 +114,50 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do // this: the client refuses to SEND a single-table frame over the cap, and // that data frame is always larger than the bare catch-up entry.) - CapShrinkHandler handler = new CapShrinkHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - handler.setServer(server); - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry - LineSenderException terminal = null; - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); - try { - sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); - sender.flush(); - // The terminal latches on the I/O thread once the reconnect's - // catch-up hits the oversized entry; it surfaces to the producer - // on a subsequent flush. Poll a bounded time for it. The polling - // rows use a small symbol that fits the shrunk cap, so the - // producer-side cap check never fires and flush() surfaces the - // I/O thread's catch-up terminal via checkError. - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { + assertMemoryLeak(() -> { + CapShrinkHandler handler = new CapShrinkHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + handler.setServer(server); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry + LineSenderException terminal = null; + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + try { + sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); + sender.flush(); + // The terminal latches on the I/O thread once the reconnect's + // catch-up hits the oversized entry; it surfaces to the producer + // on a subsequent flush. Poll a bounded time for it. The polling + // rows use a small symbol that fits the shrunk cap, so the + // producer-side cap check never fires and flush() surfaces the + // I/O thread's catch-up terminal via checkError. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { try { - sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); - sender.flush(); - Thread.sleep(20); + sender.close(); } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - sender.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; + if (terminal == null) { + terminal = e; + } } } + Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); + Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up")); } - Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); - Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), - terminal.getMessage().contains("during catch-up")); - } + }); } @Test @@ -163,48 +170,50 @@ public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Excep // into several contiguous zero-table frames that the fresh server stitches // back into a complete, gap-free dictionary. final int symbolCount = 40; - SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - // One row per flush so each frame stays under the 160-byte cap; the - // sent dictionary still accumulates all 40 symbols on connection 1. - for (int i = 0; i < symbolCount; i++) { - sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + assertMemoryLeak(() -> { + SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // One row per flush so each frame stays under the 160-byte cap; the + // sent dictionary still accumulates all 40 symbols on connection 1. + for (int i = 0; i < symbolCount; i++) { + sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + sender.flush(); + } + // Wait until connection 1 has learned every symbol, so the sender's + // sent-dictionary mirror (the catch-up source) holds all of them. + waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); + + // Wait until the server has actually closed connection 1 before + // sending batch 2, so batch 2 drives the reconnect + split catch-up. + waitFor(() -> handler.conn1Closed, 5_000); + + sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= symbolCount + 1, 10_000); } - // Wait until connection 1 has learned every symbol, so the sender's - // sent-dictionary mirror (the catch-up source) holds all of them. - waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); - - // Let the I/O loop observe the server-side close before the next - // batch, so batch 2 drives the reconnect + split catch-up. - Thread.sleep(200); - - sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); - sender.flush(); - waitFor(() -> handler.connectionsAccepted.get() >= 2 - && handler.dictFor(2).size() >= symbolCount + 1, 10_000); - } - // Connection 2's dictionary, rebuilt purely from the frames it received, - // must hold every symbol contiguously with no null gap -- the split - // catch-up frames reassemble exactly. - List conn2 = handler.dictFor(2); - Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); - for (int i = 0; i <= symbolCount; i++) { - Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + // Connection 2's dictionary, rebuilt purely from the frames it received, + // must hold every symbol contiguously with no null gap -- the split + // catch-up frames reassemble exactly. + List conn2 = handler.dictFor(2); + Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); + for (int i = 0; i <= symbolCount; i++) { + Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + } + // The catch-up had to span more than one zero-table frame to stay under + // the advertised cap -- that split is the behaviour under test. + Assert.assertTrue("catch-up split into multiple frames (saw " + + handler.zeroTableFramesOnConn2 + ")", + handler.zeroTableFramesOnConn2 >= 2); } - // The catch-up had to span more than one zero-table frame to stay under - // the advertised cap -- that split is the behaviour under test. - Assert.assertTrue("catch-up split into multiple frames (saw " - + handler.zeroTableFramesOnConn2 + ")", - handler.zeroTableFramesOnConn2 >= 2); - } + }); } private static String symbolName(int i) { @@ -252,6 +261,10 @@ private interface BoolCondition { */ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + // Set once the server has closed connection 1. A test waits on this + // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot + // race into connection 1's pre-close window and must land on the reconnect. + volatile boolean conn1Closed; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -284,6 +297,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (connNumber == 1) { Thread.sleep(50); client.close(); + conn1Closed = true; } } catch (IOException | InterruptedException e) { Thread.currentThread().interrupt(); @@ -377,6 +391,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien */ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + // Set once the server has closed connection 1 (see CatchUpHandler.conn1Closed). + volatile boolean conn1Closed; volatile int zeroTableFramesOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private final int dropConn1AtDictSize; @@ -417,6 +433,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien conn1Dropped = true; Thread.sleep(50); client.close(); + conn1Closed = true; } } catch (IOException | InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 46c2a66b..8d1bd24a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -46,6 +46,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * End-to-end recovery for the delta symbol dictionary in store-and-forward mode. *

@@ -79,131 +81,149 @@ public void tearDown() { @Test public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception { - // Phase 1: silent server (no acks). Sender 1 writes symbol rows and - // close-fast (no drain), leaving unacked delta frames + a persisted - // dictionary in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - - String pad = TestUtils.repeat("x", 64); - String cfg = "ws::addr=localhost:" + port - + ";sf_dir=" + sfDir - + ";sf_max_bytes=4096" - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < ROWS; i++) { - s1.table("m") - .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) - .stringColumn("p", pad) - .longColumn("v", i) - .atNow(); - s1.flush(); + assertMemoryLeak(() -> { + // Phase 1: silent server (no acks). Sender 1 writes symbol rows and + // close-fast (no drain), leaving unacked delta frames + a persisted + // dictionary in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String pad = TestUtils.repeat("x", 64); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .stringColumn("p", pad) + .longColumn("v", i) + .atNow(); + s1.flush(); + } } } - } - // Ack a prefix so recovery does NOT replay from the self-sufficient head. - // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the - // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN - // DISTINCT_SYMBOLS onward -- frames whose delta starts at - // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle - // reuse existing ids). The early ids those frames reference then exist - // ONLY in the persisted dictionary, so the reconstructed dictionary below - // is complete solely because the catch-up frame re-registered them. That - // pins the content assertions to the catch-up: without it (or with a - // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 - // and the per-id checks would fail. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); - - // Phase 2: fresh server that reconstructs its per-connection dictionary - // from the delta sections. Sender 2 recovers the slot and replays. - DictReconstructingHandler handler = new DictReconstructingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender ignored = Sender.fromConfig(cfg)) { - long deadline = System.currentTimeMillis() + 5_000; - while (System.currentTimeMillis() < deadline - && handler.maxDictSize() < DISTINCT_SYMBOLS) { - Thread.sleep(20); + // Ack a prefix so recovery does NOT replay from the self-sufficient head. + // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the + // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN + // DISTINCT_SYMBOLS onward -- frames whose delta starts at + // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle + // reuse existing ids). The early ids those frames reference then exist + // ONLY in the persisted dictionary, so the reconstructed dictionary below + // is complete solely because the catch-up frame re-registered them. That + // pins the content assertions to the catch-up: without it (or with a + // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 + // and the per-id checks would fail. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); + + // Phase 2: fresh server that reconstructs its per-connection dictionary + // from the delta sections. Sender 2 recovers the slot and replays. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } } - } - // The recovering sender must have re-registered the dictionary via a - // catch-up (0-table) frame before replaying delta frames. - Assert.assertTrue("recovery sent a full-dictionary catch-up frame", - handler.sawCatchUpFrame); - // The reconstructed dictionary must be complete and gap-free: exactly - // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. - List dict = handler.dictSnapshot(); - Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); - for (int i = 0; i < DISTINCT_SYMBOLS; i++) { - Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + // The recovering sender must have re-registered the dictionary via a + // catch-up (0-table) frame before replaying delta frames. + Assert.assertTrue("recovery sent a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + // The reconstructed dictionary must be complete and gap-free: exactly + // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. + List dict = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + } } - } + }); } @Test public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { - // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. - // Silent server + close-fast leaves all frames unacked in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. + // Silent server + close-fast leaves all frames unacked in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } } } - } - // Simulate a host/power crash: the segment frames survive but the persisted - // dictionary is lost, and the ack watermark was left mid-stream. Truncate - // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at - // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - java.nio.file.Path dict = slot.resolve(".symbol-dict"); - byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); - java.nio.file.Files.write(dict, header); - writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: recover against a fresh counting server. The replay guard must - // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally - // rather than send a gapped frame that would corrupt the table. - CountingHandler handler = new CountingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); - try { - Thread.sleep(1_000); // let the I/O loop attempt replay and hit the guard - } finally { + // Simulate a host/power crash: the segment frames survive but the persisted + // dictionary is lost, and the ack watermark was left mid-stream. Truncate + // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at + // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The replay guard must + // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); try { - s2.close(); - } catch (LineSenderException e) { - terminal = e; + // Poll for the replay guard to fire (it recordFatal's on the I/O + // thread); flush() surfaces the latched terminal to the producer. + // A bounded poll replaces a fixed sleep and captures it as soon as + // it fires; close() below is the fallback if it surfaces only there. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } } + Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", + 0, handler.frames.get()); + Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); } - Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", - 0, handler.frames.get()); - Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("symbol dictionary is incomplete")); - } + }); } private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 10bd1e6b..af30da83 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -38,6 +38,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * Pins down how the symbol dictionary is framed on the wire. *

@@ -64,44 +66,48 @@ public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict // so a recovered/orphan-drained slot can rebuild it. Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient"); - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - // The engine places slot files under sf_dir/ (default "default"). - Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - - // Check the persisted dictionary while the sender is live: a - // fully-drained close intentionally unlinks it (slot cleanup). - Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); - byte[] dict = Files.readAllBytes(dictFile); - Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); - Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); - } + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // The engine places slot files under sf_dir/ (default "default"). + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check the persisted dictionary while the sender is live: a + // fully-drained close intentionally unlinks it (slot cleanup). + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); + } - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); - // batch 2 ships ONLY beta as a delta from id 1. - Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", - 1, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", - 1, readVarint(b2, DELTA_START_OFFSET + 1)); + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); + // batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } finally { rmDir(sfDir); } @@ -121,43 +127,47 @@ public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Path dictPath = sfDir.resolve("default").resolve(".symbol-dict"); Files.createDirectories(dictPath); // a directory, not a file Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - } + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } - // The planted directory is untouched -- the dictionary never opened, - // so delta encoding stayed disabled. - Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", - Files.isDirectory(dictPath)); - - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - // Full-dict fallback: BOTH batches start at id 0, and batch 2 re-ships - // the WHOLE dictionary (alpha + beta), NOT a monotonic delta (which - // would be deltaStart=1, deltaCount=1 as in the test above). - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", - 1, readVarint(b1, DELTA_START_OFFSET + 1)); - Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", - 0, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", - 2, readVarint(b2, DELTA_START_OFFSET + 1)); + // The planted directory is untouched -- the dictionary never + // opened, so delta encoding stayed disabled. + Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", + Files.isDirectory(dictPath)); + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Full-dict fallback: BOTH batches start at id 0, and batch 2 + // re-ships the WHOLE dictionary (alpha + beta), NOT a monotonic + // delta (which would be deltaStart=1, deltaCount=1 as above). + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", + 0, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", + 2, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } finally { rmDir(sfDir); } @@ -181,38 +191,40 @@ private static boolean containsUtf8(byte[] haystack, String needle) { public void testMemoryModeShipsMonotonicDelta() throws Exception { // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries // only "beta" as a delta starting at id 1, not the whole dictionary. - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - } + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - // Batch 1 introduces alpha at id 0. - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", - 1, readVarint(b1, DELTA_START_OFFSET + 1)); - - // Batch 2 ships ONLY beta as a delta from id 1. - Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", - 1, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", - 1, readVarint(b2, DELTA_START_OFFSET + 1)); - } + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Batch 1 introduces alpha at id 0. + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + + // Batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } private static int readVarint(byte[] buf, int offset) { From 27aad9edc4d4254ff5ce87bf66c75735cc9d8d0b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:38:08 +0100 Subject: [PATCH 14/78] Cover the split-batch delta contract flushPendingRowsSplit fires when one flush's encoded size exceeds the server's batch cap: it emits one frame per table. The first frame must carry the whole batch's symbol-dict delta and advance the baseline, and the remaining frames must carry an empty delta that only references ids the first frame already registered -- otherwise a fresh server would see dangling symbol ids. No test drove that producer-side split. Add a test that buffers two padded tables into one flush under a small advertised cap, so the batch splits, and asserts the first frame ships deltaStart=0/deltaCount=2 while the second ships deltaStart=2/deltaCount=0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index af30da83..cb2b0118 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -227,6 +227,52 @@ public void testMemoryModeShipsMonotonicDelta() throws Exception { }); } + @Test + public void testSplitBatchShipsDeltaOnFirstFrameOnly() throws Exception { + // A single flush whose encoded size exceeds the server's batch cap is + // split into one frame per table (flushPendingRowsSplit). The FIRST split + // frame carries the whole batch's symbol-dict delta and advances the + // baseline; the remaining frames carry an EMPTY delta and only reference + // ids the first frame already registered. A regression that shipped each + // table's own symbols (wrong deltaStart) would dangle ids on the server. + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Padding inflates each table past half the cap, so the combined + // two-table message exceeds it while each single-table frame fits. + String pad = new String(new char[60]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Buffer TWO tables (symbols "a" id 0, "b" id 1), then ONE flush. + sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("the oversized two-table batch must split into 2 frames", + 2, handler.batches.size()); + byte[] f1 = handler.batches.get(0); + byte[] f2 = handler.batches.get(1); + + // First split frame ships the whole batch's dictionary (a + b). + Assert.assertEquals("first split frame deltaStart must be 0", + 0, readVarint(f1, DELTA_START_OFFSET)); + Assert.assertEquals("first split frame ships both new symbols", + 2, readVarint(f1, DELTA_START_OFFSET + 1)); + // Second split frame carries an empty delta above the advanced baseline. + Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)", + 2, readVarint(f2, DELTA_START_OFFSET)); + Assert.assertEquals("second split frame carries no new symbols", + 0, readVarint(f2, DELTA_START_OFFSET + 1)); + } + }); + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; From 034d8f30c780692c612dd6b93498eb4b36d2e8ad Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 23:12:35 +0100 Subject: [PATCH 15/78] Accumulate the tail of a partial-overlap delta into the mirror accumulateSentDict dropped a frame entirely whenever deltaStart != sentDictCount. A delta that overlaps the mirror tip and extends past it (deltaStart < sentDictCount < deltaStart+deltaCount) was therefore discarded whole -- the new tail symbols never entered the mirror, which would leave a later reconnect catch-up incomplete and shift server-side ids. The producer only ever emits strictly contiguous, non-overlapping deltas, so this is currently unreachable, but it is load-bearing correctness resting on an invariant enforced elsewhere. Handle the overlap: skip the already-held prefix [deltaStart, sentDictCount) and copy only the new tail [sentDictCount, deltaStart+deltaCount). The steady-state case (deltaStart == sentDictCount) has skip == 0, so it is unchanged and free. A gap (deltaStart > sentDictCount, which the torn-dictionary guard rejects before send) now bails explicitly rather than implicitly. Also document that the I/O-thread mirror is a second, native copy of the dictionary (the producer's GlobalSymbolDictionary already holds the same symbols as Java Strings) -- so a memory-mode connection's steady-state dictionary footprint is ~2x the symbol set, an intentional cost of the reconnect-catch-up capability. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 9f41e1e6..3e14de02 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -193,6 +193,13 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // on reconnect it can re-register the whole dictionary on the fresh server // (which discards its dictionary on every disconnect) before replaying frames // whose deltas start above id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed + // for the connection's lifetime (a reconnect may need the whole dictionary at + // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; private long sentDictBytesAddr; private int sentDictBytesCapacity; @@ -2025,15 +2032,38 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); long deltaCount = readVarintAt(p, limit); p = varintEnd; - // Only a delta that extends exactly from our current tip introduces new - // symbols. deltaStart < sentDictCount is a replay/overlap we already hold; - // deltaStart > sentDictCount is rejected before send by the torn-dictionary - // guard in trySendOne, so it never reaches here. - if (deltaCount <= 0 || deltaStart != sentDictCount) { + // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this + // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the + // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases: + // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard + // rejects it before send, so it never reaches here; bail defensively + // rather than accumulate past a hole. + // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold -- + // nothing new. + // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail. + // deltaStart == sentDictCount is the steady-state case (skip == 0). + // Handling the partial overlap explicitly -- rather than dropping the whole + // frame whenever deltaStart != sentDictCount -- keeps the mirror complete + // even if a future producer ever emits a delta that overlaps the tip; + // silently dropping the new tail would leave the reconnect catch-up + // incomplete and shift server-side ids. + long deltaEnd = deltaStart + deltaCount; + if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { return; } + // Walk past the already-held prefix [deltaStart, sentDictCount), then copy + // the new tail [sentDictCount, deltaEnd). + int skip = sentDictCount - deltaStart; + for (int i = 0; i < skip; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + return; // malformed -- bail rather than corrupt the mirror + } + } long regionStart = p; - for (long i = 0; i < deltaCount; i++) { + long newCount = deltaEnd - sentDictCount; + for (long i = 0; i < newCount; i++) { long len = readVarintAt(p, limit); p = varintEnd + len; if (p > limit) { @@ -2050,7 +2080,7 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; - sentDictCount += (int) deltaCount; + sentDictCount += (int) newCount; } private void ensureSentDictCapacity(long required) { From 41fddb11cd409af8a7452c46a220d702be909c80 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 23:12:46 +0100 Subject: [PATCH 16/78] Guard against persisted-dict duplication on a failed publish Regression test for the write-ahead persist path: persistNewSymbolsBefore- Publish runs before the frame is published (sealAndSwapBuffer -> appendBlocking). If publish fails after the persist -- here PAYLOAD_TOO_LARGE (a frame bigger than the SF segment), a backpressure deadline in production -- the symbols are already on disk but sentMaxSymbolId is not advanced and the rows stay buffered, so a retry re-runs the persist. The fix keys the persist range off pd.size() (idempotent); this pins it. The test drives one new-symbol row whose padded frame exceeds a 1 KB segment, flushes it twice (both fail to publish), then asserts the persisted .symbol-dict holds the symbol exactly once. Reverting the fix to sentMaxSymbolId+1 fails it with size 2 -- the duplicate that shifts every later global id and silently misattributes symbol values on recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictRecoveryTest.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 8d1bd24a..143686b4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -26,6 +26,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -226,6 +227,70 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { + // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs + // BEFORE the frame is published (sealAndSwapBuffer -> appendBlocking). If + // publish fails (here PAYLOAD_TOO_LARGE, a frame bigger than the SF + // segment; a backpressure deadline in production), the frame's symbols are + // already on disk but sentMaxSymbolId is NOT advanced and the rows stay + // buffered -- so a retry re-runs the persist. Keying the persist range off + // pd.size() (not sentMaxSymbolId+1) makes it idempotent. Before that fix, + // the retry appended the symbol a SECOND time, breaking the dense + // id->position invariant; on recovery every later global id shifts by one + // and symbol column values are silently misattributed. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + // Small segment; a heavily padded row's frame cannot fit, so + // appendBlocking throws PAYLOAD_TOO_LARGE deterministically -- no + // backpressure timing needed. The server never acks (SilentHandler). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;"; + String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment + Sender sender = Sender.fromConfig(cfg); + try { + // Buffer ONE new-symbol row, then flush it TWICE. Each flush + // runs the write-ahead persist and then fails to publish; the + // failed flush leaves the row buffered, so the second flush is + // the retry that (pre-fix) duplicated the persisted symbol. + sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow(); + for (int attempt = 0; attempt < 2; attempt++) { + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + // frame too large -- expected on every attempt + } + } + + // The persisted dictionary must hold "s0" EXACTLY ONCE. + // Pre-fix, the retry duplicated it (size == 2). + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol", + 1, pd.size()); + Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized row and + // fails again (PAYLOAD_TOO_LARGE); expected here and not + // what we assert. close() still runs its resource cleanup, + // so no native memory leaks. + } + } + } + }); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); From 0ef96afbcf840540c8b45a7ba07c7a33cf750a1e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:43:35 +0100 Subject: [PATCH 17/78] Cover the reconnect catch-up ACK alignment setWireBaselineWithCatchUp anchors fsnAtZero = replayStart - catchUpFrames so every catch-up frame maps to an already-acked FSN. Dropping the - catchUpFrames term is silent data loss: a server ACK for a catch-up frame then translates to an FSN at or above replayStart and trims a not-yet-delivered data frame from the store-and-forward log. The existing catch-up tests reconstruct the dictionary from wire bytes and never assert ACK/trim accounting, so they were blind to this line; the enterprise SqlFailoverQwpClientLosslessTest ingests no symbols and never enters the catch-up path at all. CursorWebSocketSendLoopCatchUpAlignmentTest drives the catch-up against a stub client and asserts the catch-up frame's OK leaves the real engine's ackedFsn untouched, for both a single catch-up frame and a split (multi-frame) catch-up. Reverting the - catchUpFrames term fails both. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 324 ++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java new file mode 100644 index 00000000..ed337f00 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -0,0 +1,324 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + +/** + * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in + * {@link CursorWebSocketSendLoop#setWireBaselineWithCatchUp}. + *

+ * On a fresh connection the loop re-registers the whole dictionary with a + * catch-up frame BEFORE replaying data frames. Each catch-up frame consumes a + * wire sequence, so the loop anchors {@code fsnAtZero = replayStart - catchUpFrames} + * to keep every catch-up frame mapped to an already-acked FSN. Dropping the + * {@code - catchUpFrames} term is silent data loss: a server ACK for a catch-up + * frame then translates through {@code engine.acknowledge(fsnAtZero + wireSeq)} + * to an FSN at or above {@code replayStart}, trimming a not-yet-delivered data + * frame from the store-and-forward log. + *

+ * The loop is constructed but never {@link CursorWebSocketSendLoop#start started}; + * the catch-up runs against a stub {@link WebSocketClient} that counts frames, and + * the OK is delivered straight into the inner {@code ResponseHandler} -- the same + * white-box idiom {@code CursorWebSocketSendLoopDurableAckTest} uses, because + * {@code setWireBaselineWithCatchUp} and the wire ports have no public entry point. + * {@link CursorSendEngine#ackedFsn()} is the authoritative trim watermark asserted + * against. + */ +public class CursorWebSocketSendLoopCatchUpAlignmentTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-cursor-catchup-" + System.nanoTime()).toString(); + assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) return; + long find = Files.findFirst(tmpDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(tmpDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(tmpDir); + } + + @Test + public void testCatchUpFrameAckDoesNotAdvanceTrimWatermark() throws Exception { + // Single catch-up frame (server advertises no cap). Two frames were + // acked before the reconnect (ackedFsn=1), FSN 2 is unacked. The catch-up + // frame's OK must NOT advance the watermark past 1 -- it carries no data, + // only the dictionary the fresh server needs before replay. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); // 0 => no cap => one frame + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 3); // FSN 0,1,2 published + engine.acknowledge(1); // ackedFsn=1 => replayStart=2, FSN 2 still unacked + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "s0", "s1"); // sentDictCount=2 => catch-up fires + long replayStart = engine.ackedFsn() + 1L; // = 2 + + invokeSetWireBaselineWithCatchUp(loop, replayStart); + + assertEquals("whole dictionary fits one frame under no cap", + 1, client.framesSent); + + // Behavioural (the harm): the catch-up frame (wire seq 0) is + // OK'd by the fresh server. It carries no data, so it must + // resolve to an already-acked FSN and leave the trim watermark + // untouched -- advancing it would trim the undelivered FSN 2. + deliverOk(loop, 0); + assertEquals("catch-up frame ACK must not advance the trim watermark " + + "(would trim an undelivered data frame -> silent data loss)", + 1L, engine.ackedFsn()); + // Mechanism: the catch-up frames are anchored below replayStart. + assertEquals("fsnAtZero must be anchored catchUpFrames below replayStart", + replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + } finally { + loop.close(); // frees the seeded mirror + the stub client's buffers + } + } + }); + } + + @Test + public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Exception { + // A small advertised cap splits the dictionary across several catch-up + // frames, so the fsnAtZero offset must subtract the full frame count. Ack + // the LAST catch-up wire sequence: it still maps below replayStart. With + // the offset dropped it would translate to replayStart+1 and over-trim. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(40); // budget 12 => one 11-byte symbol per frame + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 5); // FSN 0..4 published + engine.acknowledge(2); // ackedFsn=2 => replayStart=3, FSN 3,4 unacked + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "symbol0000", "symbol0001"); // 11 bytes each -> two frames + long replayStart = engine.ackedFsn() + 1L; // = 3 + + invokeSetWireBaselineWithCatchUp(loop, replayStart); + + assertEquals("cap must split the two symbols across two frames", + 2, client.framesSent); + + // ACK the highest catch-up wire sequence (the last catch-up + // frame). It too must map below replayStart -- with the offset + // dropped it translates to replayStart+1 and over-trims. + deliverOk(loop, client.framesSent - 1); + assertEquals("no catch-up frame ACK may advance the trim watermark", + 2L, engine.ackedFsn()); + assertEquals("fsnAtZero must subtract the full split frame count", + replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + } finally { + loop.close(); + } + } + }); + } + + private static void appendFrames(CursorSendEngine engine, int count) { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + byte[] payload = "frame-bytes-padd".getBytes(StandardCharsets.US_ASCII); + for (int i = 0; i < payload.length; i++) { + Unsafe.getUnsafe().putByte(buf + i, payload[i]); + } + for (int i = 0; i < count; i++) { + engine.appendBlocking(buf, 16); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response + // handler, mimicking the server acking a catch-up frame (which carries no tables). + private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception { + int size = 11; // status(1) + sequence(8) + tableCount(2) + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK); + Unsafe.getUnsafe().putLong(ptr + 1, wireSeq); + Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); + Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler"); + f.setAccessible(true); + Object handler = f.get(loop); + Method m = handler.getClass().getDeclaredMethod("onBinaryMessage", long.class, int.class); + m.setAccessible(true); + m.invoke(handler, ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loop, long replayStart) throws Exception { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("setWireBaselineWithCatchUp", long.class); + m.setAccessible(true); + m.invoke(loop, replayStart); + } + + private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { + return new CursorWebSocketSendLoop( + client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + throw new UnsupportedOperationException("test loop is never started"); + }, + 5_000L, 100L, 5_000L, false); + } + + private CursorSendEngine newEngine() { + return new CursorSendEngine(tmpDir, 16384); + } + + private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getLong(loop); + } + + // Populates the loop's native sent-dictionary mirror with {@code symbols} in + // the on-wire [len varint][utf8] layout, so setWireBaselineWithCatchUp sees a + // non-empty dictionary to re-register. loop.close() frees it. + private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) throws Exception { + int total = 0; + for (String s : symbols) { + int len = s.getBytes(StandardCharsets.UTF_8).length; + total += varintSize(len) + len; + } + long addr = Unsafe.malloc(total, MemoryTag.NATIVE_DEFAULT); + long p = addr; + for (String s : symbols) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + p = writeVarint(p, bytes.length); + for (byte b : bytes) { + Unsafe.getUnsafe().putByte(p++, b); + } + } + setField(loop, "sentDictBytesAddr", addr); + setIntField(loop, "sentDictBytesCapacity", total); + setIntField(loop, "sentDictBytesLen", total); + setIntField(loop, "sentDictCount", symbols.length); + } + + private static void setField(Object target, String name, long value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setLong(target, value); + } + + private static void setIntField(Object target, String name, int value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setInt(target, value); + } + + private static int varintSize(long value) { + int n = 1; + while (value > 0x7F) { + value >>>= 7; + n++; + } + return n; + } + + private static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + + // Stub transport: completes no real I/O. getServerMaxBatchSize drives the + // catch-up split; sendBinary counts the frames the catch-up emitted. + private static final class CatchUpCapturingClient extends WebSocketClient { + private final int cap; + private int framesSent; + + CatchUpCapturingClient(int cap) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.cap = cap; + } + + @Override + public int getServerMaxBatchSize() { + return cap; + } + + @Override + public int getServerQwpVersion() { + return 1; + } + + @Override + public void sendBinary(long dataPtr, int length) { + framesSent++; + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } +} From ff79b72f9c798b79e8a558314b34932277209d8c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:47:08 +0100 Subject: [PATCH 18/78] Cover the retriable catch-up send containment sendCatchUpChunk throws CatchUpSendException on a transient wire failure instead of calling fail(). From inside the catch-up fail() re-enters connectLoop -- desyncing the fsnAtZero/nextWireSeq wire mapping (a later ACK then trims un-acked store-and-forward frames), or overflowing the stack on a flapping connection -- turning a transient outage into a hard failure. Only the oversized-entry (non-retriable) terminal was covered; the retriable path had no test. testTransientCatchUpSendFailureIsRetriableNotTerminal drives the catch-up against a stub whose sendBinary throws, and asserts the failure surfaces as a retriable CatchUpSendException and leaves the producer-facing error latch clear. Reverting the throw to fail() fails it. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index ed337f00..9b2195d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -39,11 +39,13 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in @@ -173,6 +175,42 @@ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Excepti }); } + @Test + public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Exception { + // A transient wire failure WHILE shipping the catch-up (the fresh + // connection drops mid-handshake) must surface as a retriable + // CatchUpSendException for the reconnect loop to handle -- it must NOT + // call fail(). From inside the catch-up fail() re-enters connectLoop + // (corrupting the fsnAtZero/nextWireSeq mapping, or overflowing the stack + // on a flapping connection) or, with no reconnect attempt reachable, + // latches a terminal -- turning a transient outage into a hard failure and + // breaking store-and-forward. Only the oversized-entry (non-retriable) + // terminal was covered; this pins the retriable path. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); // sendBinary throws + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + engine.acknowledge(0); // ackedFsn=0 => a real unacked frame exists behind the catch-up + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "s0", "s1"); // non-empty dict => catch-up fires and hits the failing send + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("a transient catch-up send failure must raise a retriable " + + "CatchUpSendException, not be swallowed into fail()/a terminal"); + } catch (InvocationTargetException e) { + assertEquals("transient catch-up send failure must surface as CatchUpSendException", + "CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + // Retriable, not terminal: the producer-facing error latch stays clear. + loop.checkError(); + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { @@ -288,14 +326,22 @@ private static long writeVarint(long addr, long value) { } // Stub transport: completes no real I/O. getServerMaxBatchSize drives the - // catch-up split; sendBinary counts the frames the catch-up emitted. + // catch-up split; sendBinary counts the frames the catch-up emitted, or -- + // when throwOnSend is set -- raises a transient wire error to model the fresh + // connection dropping mid-catch-up. private static final class CatchUpCapturingClient extends WebSocketClient { private final int cap; + private final boolean throwOnSend; private int framesSent; CatchUpCapturingClient(int cap) { + this(cap, false); + } + + CatchUpCapturingClient(int cap, boolean throwOnSend) { super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); this.cap = cap; + this.throwOnSend = throwOnSend; } @Override @@ -310,6 +356,9 @@ public int getServerQwpVersion() { @Override public void sendBinary(long dataPtr, int length) { + if (throwOnSend) { + throw new RuntimeException("transient wire failure during catch-up"); + } framesSent++; } From 8fd9ad19a2860b47097b8fe861df2f41c5027474 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:55:59 +0100 Subject: [PATCH 19/78] Tidy catch-up comments and harden two edges Four minor cleanups on the delta symbol-dictionary catch-up, all behaviour-preserving on every reachable path: - The sentDict* field comment said the catch-up mirror is memory-mode only; it is also seeded and used in disk mode on a recovered / orphan-drained slot. Corrected. - positionCursorAt's javadoc said it runs after nextWireSeq was reset to 0, but the catch-up path leaves nextWireSeq past the frames it emitted. Corrected to describe setWireBaselineWithCatchUp anchoring the wire baseline; the method only moves the byte cursor. - The recovery-seed constructor set sentDictCount = pd.size() outside the loadedEntriesLen > 0 block. A recovered slot always has entries when size > 0, so the result is unchanged, but coupling the count to the mirror bytes stops sentDictCount ever claiming symbols the mirror does not hold. - sendDictCatchUp used Integer.MAX_VALUE as the no-cap per-frame budget, so sendCatchUpChunk's int frameLen could overflow on a multi-GB dictionary. Bound it by MAX_SENT_DICT_BYTES, the same ceiling ensureSentDictCapacity enforces. Unreachable at real cardinality (~200M+ symbols); defensive. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3e14de02..7a9086d8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -186,7 +186,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); - // Delta symbol dictionary catch-up state (memory-mode only; see swapClient). + // Delta symbol dictionary catch-up state (see swapClient). Active in memory + // mode, and in disk mode on a recovered / orphan-drained slot (the constructor + // seeds sentDict* from the persisted dictionary there). // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that @@ -532,8 +534,12 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, ensureSentDictCapacity(len); Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold. A recovered slot + // always has loadedEntriesLen > 0 when size > 0, so this is the + // same result -- it just makes the coupling explicit. + sentDictCount = pd.size(); } - sentDictCount = pd.size(); } } this.fsnAtZero = fsnAtZero; @@ -1784,8 +1790,11 @@ private void ioLoop() { /** * Walk the engine's segments to find the one containing {@code targetFsn}, * and set {@code sendOffset} to the byte offset of that frame within it. - * This is called at startup and after every reconnect, after fsnAtZero has - * already been reset to {@code targetFsn} and nextWireSeq to 0. + * This is called at startup and after every reconnect, once + * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline + * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq} + * past the catch-up frames it emitted. This method only positions the byte + * cursor at {@code targetFsn}; it does not touch the wire mapping. *

* If {@code targetFsn} is already published, the method positions the byte * cursor exactly at that frame. If {@code targetFsn} is not published yet, @@ -2133,8 +2142,13 @@ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); // Symbol-bytes budget per frame, leaving room for the 12-byte header and // the two delta-section varints. cap <= 0 means the server advertised no - // limit -> send the whole dictionary in one frame. - int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : Integer.MAX_VALUE; + // limit -- send the whole dictionary in one frame, but still bound the + // budget by MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen + // (HEADER_SIZE + varints + symbolsLen) cannot overflow on a pathological + // multi-GB dictionary (unreachable at real cardinality; defensive). + int budget = cap > 0 + ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) + : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; int framesSent = 0; int chunkStartId = 0; long chunkStartAddr = sentDictBytesAddr; From 8fcd8359089e9b177fc24706a72644a15f72def6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 02:46:13 +0100 Subject: [PATCH 20/78] Fix delta symbol-dict recovery and reconnect bugs Close three ways the delta symbol-dictionary feature could lose or corrupt data on the reconnect and store-and-forward recovery paths. Run the torn-dictionary guard unconditionally. trySendOne gated the guard on deltaDictEnabled, which CursorSendEngine reports false when a recovered disk slot cannot open its persisted dictionary (fd exhaustion, a read-only remount, ENOSPC). The recorded frames are still delta frames, so replaying them against a fresh empty-dictionary server null-padded the missing ids and silently corrupted the table. The guard now decodes the delta start for every frame and fails terminally on a gap regardless of the flag; only the sent-dictionary mirror stays gated. Stop treating a catch-up frame as the head data frame. sendCatchUpChunk advances nextWireSeq, but onClose's poison-strike gate and handleServerRejection's pre-send gate read nextWireSeq > 0 as "a data frame was sent". A transient non-orderly close or NACK after the catch-up but before the first replay frame then charged a poison strike on a frame that never left, and after a few flaps escalated a transient outage to a PROTOCOL_VIOLATION terminal that quarantines an orphan drainer. A new dataFrameSentThisConnection flag, set only after a real ring frame sends, now gates both decisions, so the drainer keeps retrying as Invariant B requires. Bound the commit message's dictionary delta to the sent watermark. sendCommitMessage skips the write-ahead persist yet encoded a delta up to currentBatchMaxSymbolId, so a symbol left in the batch by a cancelled row (cancelRow rolls back neither currentBatchMaxSymbolId nor the global registration) rode out on the commit frame without being persisted. A recovered slot then under-seeded the producer against the surviving frame and misattributed the reused id. The commit now caps the delta at sentMaxSymbolId in delta mode, giving an empty delta. Each fix carries a regression test proven to fail when the fix is reverted: a directory-shadowed .symbol-dict (guard), a close after only the catch-up (poison gate), and a cancelled-row symbol on a transactional commit (delta bound). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 19 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 100 ++++++++----- .../qwp/client/DeltaDictRecoveryTest.java | 133 ++++++++++++++++++ ...ursorWebSocketSendLoopPoisonFrameTest.java | 51 +++++++ 4 files changed, 264 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 77e85392..4a499300 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3623,10 +3623,23 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); - // A commit carries no rows and no new symbols; in delta mode its empty - // delta simply starts at the server's current dictionary size. + // A commit carries no rows, and it must also carry NO new symbols. Unlike + // the flush paths, sendCommitMessage does NOT write-ahead-persist the + // dictionary, so shipping a symbol here would put an id on the wire that a + // recovered slot cannot rebuild from the persisted .symbol-dict, diverging + // the producer dictionary from the surviving frames and silently + // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit + // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back + // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at + // what has already been sent -- and therefore already persisted. In delta + // mode pass sentMaxSymbolId, yielding an empty delta + // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep + // currentBatchMaxSymbolId so the frame stays self-sufficient. Any symbol a + // cancelled row leaked is picked up (and persisted) by the next real flush, + // whose persistNewSymbolsBeforePublish resumes from pd.size(). + int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; encoder.beginMessage(0, globalSymbolDictionary, - symbolDeltaBaseline(), currentBatchMaxSymbolId); + symbolDeltaBaseline(), commitBatchMaxId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 7a9086d8..2772cdf8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -203,6 +203,18 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // for the connection's lifetime (a reconnect may need the whole dictionary at // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up + // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies + // "the head frame was sent": onClose's poison-strike gate and + // handleServerRejection's pre-send gate key off THIS instead. Without it, a + // transient outage AFTER the catch-up but BEFORE the first data frame (a + // flapping LB/middlebox that accepts the upgrade + catch-up then closes) would + // be mistaken for a deterministic head-frame rejection and escalate to a + // PROTOCOL_VIOLATION terminal -- breaking the store-and-forward "retry a + // transient outage forever" contract. Reset per connection in + // setWireBaselineWithCatchUp; set in trySendOne after a successful send. + private boolean dataFrameSentThisConnection; private long sentDictBytesAddr; private int sentDictBytesCapacity; private int sentDictBytesLen; @@ -1980,6 +1992,11 @@ private void swapClient(WebSocketClient newClient) { * first real connection via swapClient. */ private void setWireBaselineWithCatchUp(long replayStart) { + // Fresh connection: no data frame has been sent on it yet. Reset before the + // catch-up (which sends only dictionary frames) so onClose / + // handleServerRejection can tell "only the catch-up went out" from "the + // head data frame went out". + dataFrameSentThisConnection = false; if (client != null && deltaDictEnabled && sentDictCount > 0) { this.nextWireSeq = 0L; // The catch-up may span several frames when the dictionary exceeds the @@ -2338,29 +2355,31 @@ private boolean trySendOne() { return false; // payload not fully published yet } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; - // -1 unless this is a delta frame; the guard decodes it once here and - // accumulateSentDict reuses it post-send, so the delta header is parsed - // once per frame rather than twice. - int deltaStart = -1; - if (deltaDictEnabled) { - // Torn-dictionary guard. In normal operation a delta frame's start id - // never exceeds the dictionary coverage established so far (replayed - // frames overlap the catch-up dict; fresh frames extend it - // contiguously). A gap here means the persisted dictionary was torn -- - // almost always by a host/power crash, which leaves segment frames on - // disk but loses recently-written dictionary entries (SF, like the rest - // of the store, is process-crash durable but not host-crash durable). - // Sending the frame would corrupt the table (the server would null-pad - // the missing ids), so fail terminally instead; the unreplayable data - // must be resent. - deltaStart = frameDeltaStart(frameAddr, payloadLen); - if (deltaStart > sentDictCount) { - recordFatal(new LineSenderException( - "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " - + "frame delta start " + deltaStart + " exceeds recovered dictionary size " - + sentDictCount + "; cannot replay without corrupting data -- resend required")); - return false; - } + // Torn-dictionary guard. Decode the delta start unconditionally (-1 for a + // non-delta frame); the guard MUST run even when deltaDictEnabled is false. + // A disk slot recovered with its persisted dictionary unavailable + // (PersistedSymbolDict.open() returned null -- fd exhaustion, a read-only + // remount, ENOSPC) reports deltaDictEnabled=false, yet its recorded frames + // are still DELTA frames (deltaStart > 0). Replaying those against a fresh + // empty-dictionary server would null-pad the missing ids and SILENTLY + // corrupt the table -- precisely what this guard exists to prevent -- so it + // cannot be gated on the very flag that goes false in that failure mode. In + // normal operation a delta frame's start id never exceeds the dictionary + // coverage established so far (replayed frames overlap the catch-up dict; + // fresh frames extend it contiguously), so a gap here means the recovered + // dictionary is incomplete (a host/power crash that lost recently-written + // entries, SF being process-crash but not host-crash durable). Fail + // terminally; the unreplayable data must be resent. Full-dict / fallback + // frames carry deltaStart=0 with sentDictCount=0, so 0 > 0 never + // false-positives; only the sent-dictionary mirror below stays gated on + // deltaDictEnabled. + int deltaStart = frameDeltaStart(frameAddr, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; } try { client.sendBinary(frameAddr, payloadLen); @@ -2368,7 +2387,12 @@ private boolean trySendOne() { fail(t); return false; } - if (deltaStart >= 0) { + // A real ring frame (data or commit) has now gone out on this connection, + // as opposed to only the dictionary catch-up. onClose / handleServerRejection + // key their poison-strike vs pre-send decision off this, not off nextWireSeq + // (which the catch-up advances). + dataFrameSentThisConnection = true; + if (deltaDictEnabled && deltaStart >= 0) { // Mirror the symbols this frame introduced so a later reconnect can // rebuild the whole dictionary. Idempotent on replay: a frame whose // delta we already hold advances nothing. @@ -2619,7 +2643,7 @@ public void onClose(int code, String reason) { || code == WebSocketCloseCode.GOING_AWAY; LineSenderException cause = new LineSenderException( "WebSocket closed by server: code=" + code + " reason=" + reason); - if (!orderly && nextWireSeq > 0) { + if (!orderly && dataFrameSentThisConnection) { if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) { haltOnPoisonedFrame("ws-close[" + code + ' ' + WebSocketCloseCode.describe(code) + "]: " + reason, @@ -2726,17 +2750,21 @@ private void handleServerRejection(long wireSeq) { // value is only used to attribute an FSN to the error report -- // a rejection never advances the watermark. long highestSent = nextWireSeq - 1L; - if (highestSent < 0L) { - // Pre-send rejection: server emitted an error frame before - // we sent anything on this connection (typical after a - // fresh swapClient — auth failure, server-initiated halt, - // etc.). The server-named wireSeq does not correspond to - // any frame we sent, so clamping it to 0 and acknowledging - // fsnAtZero would silently advance ackedFsn past a real - // unsent batch (fsnAtZero == ackedFsn + 1 right after a - // swap). Skip the watermark advance entirely; still surface - // the error so the user's handler sees it and HALT errors - // remain producer-observable. + if (!dataFrameSentThisConnection) { + // Pre-send rejection: the server emitted an error frame before we + // sent any DATA frame on this connection (typical after a fresh + // swapClient -- auth failure, server-initiated halt, or a rejection + // of the dictionary catch-up itself). nextWireSeq may be > 0 here + // because the catch-up consumed wire sequences, so this keys off + // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a + // transient NACK of a catch-up frame would take the post-send + // poison-strike path and could escalate a transient outage to a + // terminal. The server-named wireSeq does not correspond to any + // data frame we sent, so clamping it to 0 and acknowledging + // fsnAtZero would silently advance ackedFsn past a real unsent + // batch. Skip the watermark advance entirely; still surface the + // error so the user's handler sees it and HALT errors remain + // producer-observable. handlePreSendRejection(wireSeq, status, category, policy); return; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 143686b4..81fd8676 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,139 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { + // C1 regression: when a recovered disk slot's persisted dictionary cannot be + // OPENED (fd exhaustion, a read-only remount, ENOSPC -- simulated here by a + // .symbol-dict that is a DIRECTORY, so both openRW and openCleanRW fail), + // CursorSendEngine.isDeltaDictEnabled() returns false. The recorded frames + // are still DELTA frames, and replaying them against a fresh + // empty-dictionary server would null-pad the missing ids and SILENTLY + // corrupt the table. The torn-dictionary guard must fire regardless of + // deltaDictEnabled -- pre-fix it was gated on that very flag, so the + // corrupting frame sailed through unguarded. Unlike + // testTornDictionaryFailsCleanlyInsteadOfCorrupting (dict present but empty, + // deltaDictEnabled=true), here the dict is UNOPENABLE (deltaDictEnabled=false). + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Make the persisted dictionary UNOPENABLE: replace the .symbol-dict file + // with a directory of the same name so PersistedSymbolDict.open() returns + // null (both openRW and openCleanRW fail) and the engine reports + // deltaDictEnabled=false. Stamp the watermark at FSN 2 so replay starts + // at FSN 3 -- a frame whose delta starts at id 3, with ids 0..2 living + // only in the now-unreadable dictionary. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The guard must fire + // (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertEquals("no delta frame may be replayed when the persisted dictionary is unopenable", + 0, handler.frames.get()); + Assert.assertNotNull("an unopenable dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); + } + }); + } + + @Test + public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Exception { + // C3 regression: sendCommitMessage does NOT write-ahead-persist the + // dictionary, so its frame must carry NO new symbol. A symbol left in the + // batch by a cancelled row -- cancelRow rolls back neither + // currentBatchMaxSymbolId nor the global-dictionary registration -- must not + // ride out on the commit frame: doing so puts an id on the wire that a + // recovered slot cannot rebuild from .symbol-dict, diverging the producer + // dictionary from the surviving frames and silently misattributing reused + // ids after a crash. The commit's delta must be bounded by sentMaxSymbolId + // (empty here), not currentBatchMaxSymbolId. Memory mode suffices to observe + // the wire behaviour; close() drains every frame to the server first. + assertMemoryLeak(() -> { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Transactional + autoFlushRows=1: every completed row auto-flushes + // as a DEFERRED batch (setting hasDeferredMessages); an explicit + // flush() then emits the commit message. + Sender sender = Sender.builder("ws::addr=localhost:" + port + ";") + .transactional(true) + .autoFlushRows(1) + .build(); + try { + // Row 1 registers "a"@0 and auto-flushes it deferred. + sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow(); + // Register "b"@1 on a row that is then cancelled: "b" stays in + // the global dictionary and currentBatchMaxSymbolId advances to + // 1, but nothing persists or sends it. + sender.table("m").symbol("s", "b"); + sender.cancelRow(); + // Commit the deferred batch. The commit frame must carry an + // EMPTY delta -- NOT "b"@1. + sender.flush(); + } finally { + sender.close(); // drains every frame (incl. the commit) to the server + } + + // The server's reconstructed dictionary must hold ONLY "a". Pre-fix + // the commit shipped "b"@1, so the server saw a second symbol. + List dict = handler.dictSnapshot(); + Assert.assertEquals("commit frame must not ship the cancelled row's leaked symbol " + + "(recovery would then diverge from the persisted dictionary): " + dict, + 1, dict.size()); + Assert.assertEquals("a", dict.get(0)); + } + }); + } + @Test public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index 34ae5518..6a3d8079 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -244,6 +244,41 @@ public void testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine() throws Exception }); } + @Test + public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception { + // C2 regression: the dictionary catch-up advances nextWireSeq WITHOUT + // sending a data frame. A non-orderly close in that window -- a flapping + // LB/middlebox that completes the upgrade, accepts the catch-up, then drops + // before the first replay frame -- must be strike-EXEMPT. Keying the + // poison-strike gate off nextWireSeq > 0 (rather than + // dataFrameSentThisConnection) charges a strike on a frame that was never + // sent; MAX_REJECTIONS such closes then escalate a TRANSIENT outage to a + // PROTOCOL_VIOLATION terminal, hard-failing the producer and quarantining an + // orphan drainer -- exactly what store-and-forward's retry-forever contract + // forbids. Mirror image of testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine, + // which sends a real data frame (setSentCount) and DOES escalate. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + for (int i = 0; i < MAX_REJECTIONS + 2; i++) { + // Model the catch-up: nextWireSeq advanced, but NO data frame + // sent. swapClient resets both on every recycle, so re-apply + // before each close. Pre-fix, each of these lands a strike on the + // never-sent head frame and the loop terminals by now. + setCatchUpWireSeqOnly(loop, 2); + deliverNonOrderlyClose(loop); + } + // No strike was ever charged, so nothing escalated: the loop stays + // retriable and the producer-facing error latch is clear. + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { // A reachable, healthy server that NACKs the head frame (RETRIABLE) @@ -969,6 +1004,22 @@ private static void setSentCount(CursorWebSocketSendLoop loop, long count) throw Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); f.setAccessible(true); f.setLong(loop, count); + // A non-zero sent count models DATA frames sent on this connection. The + // poison-strike / pre-send gates key off dataFrameSentThisConnection, not + // nextWireSeq (the dictionary catch-up advances nextWireSeq without sending + // a data frame), so keep the two in sync for these white-box scenarios. + Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); + d.setAccessible(true); + d.setBoolean(loop, count > 0); + } + + // Sets ONLY nextWireSeq -- deliberately NOT dataFrameSentThisConnection -- to + // model the dictionary catch-up having advanced the wire sequence with no data + // frame sent yet. Contrast setSentCount, which sets both. + private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long count) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); + f.setAccessible(true); + f.setLong(loop, count); } private static long[] txns(long... v) { From fc8b4ba64dd0e7a52957891bcf44217e361f7774 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 03:06:18 +0100 Subject: [PATCH 21/78] Adopt recovered dictionary into the send mirror On recovery the send loop copied the persisted dictionary's loaded-entries buffer into a fresh mirror allocation and left PersistedSymbolDict holding a second copy for the engine's lifetime -- roughly twice the dictionary size in native memory on a high-cardinality recovered slot, retained long after the one-time seed. The loop now adopts that buffer as its mirror backing via takeLoadedEntries(), which transfers ownership so the dictionary no longer retains or frees it. The producer's readLoadedSymbols() is the only other consumer and runs first (setCursorEngine seeds the producer before the loop is built; the drainer has no producer consumer), guarded by an assert. Add a recover-then-continue-ingest test. A file-mode sender writes symbols and crashes; a fresh sender recovers the slot and ingests a NEW symbol. It asserts the producer continues the dictionary from the recovered size instead of colliding at id 0, exercising seedGlobalDictionaryFromPersisted, which no prior test drove past recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 12 +++- .../client/sf/cursor/PersistedSymbolDict.java | 37 ++++++++++-- .../qwp/client/DeltaDictRecoveryTest.java | 56 +++++++++++++++++++ 3 files changed, 97 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2772cdf8..f5f21207 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -543,8 +543,16 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, if (pd != null && pd.size() > 0) { int len = pd.loadedEntriesLen(); if (len > 0) { - ensureSentDictCapacity(len); - Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + // Adopt the persisted dictionary's loaded-entries buffer as the + // mirror's initial backing instead of copying it and leaving the + // dictionary to retain a second copy for the engine's lifetime. + // The producer's readLoadedSymbols() -- the only other consumer -- + // runs earlier in setCursorEngine; the drainer path has no + // producer consumer. takeLoadedEntries() transfers ownership, so + // this loop's mirror lifecycle frees it (not pd.close()). The + // buffer's allocated size equals loadedEntriesLen. + sentDictBytesAddr = pd.takeLoadedEntries(); + sentDictBytesCapacity = len; sentDictBytesLen = len; // Set the count only alongside the bytes so sentDictCount can // never claim symbols the mirror does not hold. A recovered slot diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 15179e89..dbb3593c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -94,14 +94,19 @@ public final class PersistedSymbolDict implements QuietCloseable { private static final int MAX_ENTRY_LEN = 1 << 20; private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; - // In-memory copy of the entry region [len][utf8]... exactly as on disk, - // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. Consumed once to - // seed the send loop's catch-up mirror and the producer's id map. - private final long loadedEntriesAddr; - private final int loadedEntriesLen; private long appendOffset; private boolean closed; + // In-memory copy of the entry region [len][utf8]... exactly as on disk, + // populated only when open() recovered existing entries (recovery / + // orphan-drain). Zero/empty for a freshly created file. Consumed once to seed + // the producer's id map (readLoadedSymbols) and then ADOPTED by the send loop's + // catch-up mirror (takeLoadedEntries), which takes ownership so this file no + // longer retains a second copy of the dictionary for the engine's lifetime. + private long loadedEntriesAddr; + private int loadedEntriesLen; + // True once takeLoadedEntries() handed loadedEntriesAddr to the send-loop + // mirror; guards a stale readLoadedSymbols() call (which must run first). + private boolean loadedEntriesTaken; private long scratchAddr; private int scratchCap; private int size; @@ -256,6 +261,7 @@ public int loadedEntriesLen() { * recovered. */ public ObjList readLoadedSymbols() { + assert !loadedEntriesTaken : "readLoadedSymbols() must run before takeLoadedEntries()"; ObjList out = new ObjList<>(Math.max(size, 1)); long p = loadedEntriesAddr; long limit = p + loadedEntriesLen; @@ -286,6 +292,25 @@ public int size() { return size; } + /** + * Transfers ownership of the loaded-entries buffer to the caller and returns + * its base address (0 when nothing was recovered). The send loop adopts it as + * the initial backing of its catch-up mirror rather than copying it, so this + * file stops retaining a second copy of the dictionary for the engine's + * lifetime. After this call {@link #close()} no longer frees the buffer -- the + * caller owns it. The producer's {@link #readLoadedSymbols()} is the only other + * consumer and MUST run first: {@code setCursorEngine} seeds the producer + * before the send loop is constructed, and the drainer path has no producer + * consumer. The buffer was allocated with {@link MemoryTag#NATIVE_DEFAULT}. + */ + public long takeLoadedEntries() { + long addr = loadedEntriesAddr; + loadedEntriesAddr = 0L; + loadedEntriesLen = 0; + loadedEntriesTaken = true; + return addr; + } + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 81fd8676..8174ed24 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,62 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { + // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's + // dictionary and delta baseline from the persisted .symbol-dict, so a + // recovered sender that continues ingesting assigns the NEXT id, not a + // colliding low one. Without it the new symbol reuses a recovered id and the + // fresh server sees a redefinition -> silent misattribution. No prior test + // ingests on the recovered sender. Replay is from FSN 0 (no acks), so the + // recovered frames legitimately overlap the seeded dictionary -- this also + // pins that the redefinition guard does not false-positive on normal + // recovery. + assertMemoryLeak(() -> { + // Phase 1: ingest DISTINCT_SYMBOLS symbols, silent server, close-fast -> + // unacked frames + a full persisted dictionary. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096;close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Phase 2: recover against a fresh server, then ingest a genuinely NEW + // symbol. The producer must continue at id DISTINCT_SYMBOLS. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "brand-new").longColumn("v", 99L).atNow(); + s2.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS + 1) { + Thread.sleep(20); + } + } + List dict = handler.dictSnapshot(); + Assert.assertEquals("recovered sender must continue the dictionary, not collide: " + dict, + DISTINCT_SYMBOLS + 1, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("sym-" + i, dict.get(i)); + } + Assert.assertEquals("brand-new", dict.get(DISTINCT_SYMBOLS)); + } + }); + } + @Test public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { // C1 regression: when a recovered disk slot's persisted dictionary cannot be From 8dd7723789615b679b5047ad74778c16567a264e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 09:54:44 +0100 Subject: [PATCH 22/78] Harden delta symbol-dict recovery and NACK gating Two fixes surfaced by review of the delta symbol-dictionary change. Recover oversized persisted symbols (C1). PersistedSymbolDict.openExisting rejected any entry larger than a fixed 1 MB ceiling as "oversized" and truncated the dictionary at that id, but the write path (appendSymbol/appendSymbols) caps nothing. A symbol value over 1 MB therefore persisted fine, yet a normal process-crash recovery dropped it and every higher id; the send loop's replay guard then fired a spurious "host crash / resend required" terminal -- a store-and-forward process-crash-durability violation on a reachable input (nothing caps symbol value length). Drop the fixed per-entry ceiling and bound entries only by the file length, which is the actual corruption guard and was already present. Keep the length in a long so a corrupt multi-gigabyte varint cannot wrap an int back under the check. The write and read paths now agree, so a legitimately persisted large symbol recovers. Guard the catch-up NACK pre-send gate (C2). handleServerRejection keys its pre-send branch off dataFrameSentThisConnection, because the symbol catch-up advances nextWireSeq without sending a data frame. That branch had no regression test: every existing NACK test sets both flags together, so reverting the gate to the old nextWireSeq-based predicate left the suite green. Add the server-NACK twin of testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike -- a WRITE_ERROR NACK of a catch-up frame must take the pre-send path (surface and recycle, no strike), not escalate a transient outage to a producer-fatal PROTOCOL_VIOLATION terminal. Both regression tests fail with their production line reverted and pass with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 19 +++++--- ...ursorWebSocketSendLoopPoisonFrameTest.java | 36 +++++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 45 +++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index dbb3593c..ca769181 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -89,9 +89,6 @@ public final class PersistedSymbolDict implements QuietCloseable { static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; static final byte VERSION = 1; - // Guards against a hostile/corrupt varint length driving a huge allocation - // or a runaway parse. Symbols are short; this is a generous ceiling. - private static final int MAX_ENTRY_LEN = 1 << 20; private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; private long appendOffset; @@ -337,12 +334,20 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (vr == null) { break; // torn length varint } - int entryLen = (int) vr[0]; + long entryLen = vr[0]; int next = (int) vr[1]; - if (entryLen < 0 || entryLen > MAX_ENTRY_LEN || (long) next + entryLen > len) { - break; // torn/oversized entry -- self-healing tail + // The file length is the sole sanity bound: a length that runs past + // the file is a torn/incomplete trailing entry and stops the parse + // (self-healing tail). There is deliberately NO fixed per-entry + // ceiling -- the write path (appendSymbol/appendSymbols) applies none + // either, so a legitimately persisted large symbol must recover here, + // not be truncated and then trip the send loop's "resend required" + // guard on a normal process-crash recovery. entryLen stays a long so a + // corrupt multi-gigabyte length cannot wrap an int back under the check. + if ((long) next + entryLen > len) { + break; // torn/incomplete trailing entry -- self-healing tail } - pos = next + entryLen; + pos = next + (int) entryLen; count++; } int entriesLen = pos - HEADER_SIZE; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index 6a3d8079..ab200e80 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -279,6 +279,42 @@ public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception }); } + @Test + public void testNonOrderlyRejectionAfterOnlyCatchUpDoesNotStrike() throws Exception { + // C2 regression: the server-NACK twin of + // testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike. handleServerRejection's + // pre-send gate keys off dataFrameSentThisConnection, NOT nextWireSeq -- the + // dictionary catch-up advances nextWireSeq WITHOUT sending a data frame. A + // server NACK of a catch-up frame (nextWireSeq>0, no data frame sent) must + // take the pre-send path (surface + recycle, no strike), not the post-send + // poison-strike path. Pre-fix (gate on highestSent >= 0, i.e. nextWireSeq>0) + // each catch-up NACK strikes the never-sent head frame; MAX_REJECTIONS such + // strikes escalate a TRANSIENT outage to a producer-fatal PROTOCOL_VIOLATION + // terminal -- exactly what store-and-forward's retry-forever contract forbids. + // WRITE_ERROR (RETRIABLE) is the discriminating category: it DOES accrue + // strikes on the post-send path (see testNackRecycleIsPacedAgainstHealthyServer), + // so a regressed gate escalates here and checkError() throws. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + for (int i = 0; i < MAX_REJECTIONS + 2; i++) { + // Model the catch-up: nextWireSeq advanced, but NO data frame + // sent. The pre-send recycle (swapClient) resets nextWireSeq, so + // re-apply before each NACK (mirrors the onClose twin). + setCatchUpWireSeqOnly(loop, 2); + deliverRetriableNack(loop, 1, "disk full"); + } + // No strike was ever charged, so nothing escalated: the loop stays + // retriable and the producer-facing error latch is clear. + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { // A reachable, healthy server that NACKs the head frame (RETRIABLE) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index df6bab21..d580f886 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.ObjList; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -198,6 +199,50 @@ public void testEmptySymbolRoundTrips() throws Exception { }); } + @Test + public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { + // C1 regression: the write path caps nothing, so a symbol larger than the + // old fixed 1 MB read ceiling must still recover intact. Before the fix, + // appendSymbol wrote the oversized entry but openExisting rejected it as + // "oversized", truncated the dictionary at that id (dropping it and every + // higher id), and a normal process-crash recovery then hard-failed with a + // spurious "host crash / resend required" terminal -- defeating store-and- + // forward's process-crash durability for large symbols. The file length is + // now the only bound, so the write and read paths agree. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Just over the old 1 << 20 (1 MB) ceiling. + String big = TestUtils.repeat("x", (1 << 20) + 17); + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol("before"); + d.appendSymbol(big); + d.appendSymbol("after"); + Assert.assertEquals(3, d.size()); + } finally { + d.close(); + } + + // Recovery must load ALL three; pre-fix the reopen truncated at the + // big entry and came back with size 1 (only "before" survived). + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals("large entry must survive recovery, not be truncated", + 3, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("before", s.getQuick(0)); + Assert.assertEquals(big, s.getQuick(1)); + Assert.assertEquals("after", s.getQuick(2)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { From 8853d8fd5fe922ce6b27b475b84e315dac9271e1 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 10:01:30 +0100 Subject: [PATCH 23/78] Persist symbol delta from frame, skip re-encode M1: persistNewSymbolsBeforePublish re-encoded each new symbol into a scratch buffer even though beginMessage had just written the identical [len][utf8] bytes into the frame's symbol-dict delta section. The encoder now records that entry region's offset and length; the producer writes those bytes straight to the slot .symbol-dict via a new PersistedSymbolDict.appendRawEntries, so a high-cardinality batch no longer re-encodes what the wire frame already carries. The common path (durable size == the frame's delta start id) takes the direct copy; a retry after a failed publish, where the durable size has run ahead of the wire baseline, falls back to re-encoding just the remaining suffix. The persisted bytes are byte-identical either way. M2: document, on the sendDictCatchUp oversized-entry terminal, that in a heterogeneous or rolling-cap cluster a symbol accepted under a larger cap can hit this hard stop on failover to a smaller-cap node and will not self-recover if a later node advertises a larger cap. Behavior is unchanged -- the terminal keeps the homogeneous common case livelock-free; this only records the tradeoff and the two ways to relax it (an ingest-side symbol-size bound, or a settle budget across reconnects before latching). Adds testAppendRawEntriesMatchesAppendSymbols; the file-mode persist, recovery, and failed-publish suites cover both the direct-copy and re-encode branches end to end. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketEncoder.java | 26 +++++++++ .../qwp/client/QwpWebSocketSender.java | 45 ++++++++++----- .../sf/cursor/CursorWebSocketSendLoop.java | 9 +++ .../client/sf/cursor/PersistedSymbolDict.java | 24 ++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 56 +++++++++++++++++++ 5 files changed, 146 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ced1a1b5..8013330a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -39,6 +39,14 @@ public class QwpWebSocketEncoder implements QuietCloseable { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; + // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region + // ([len][utf8]... only, without the two section varints) that beginMessage + // last wrote. Let the producer persist those bytes straight to the slot's + // .symbol-dict instead of re-encoding the same symbols (see + // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next + // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaEntriesEnd; + private int deltaEntriesStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -75,10 +83,12 @@ public void beginMessage( payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); + deltaEntriesStart = buffer.getPosition(); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } + deltaEntriesEnd = buffer.getPosition(); columnWriter.setBuffer(buffer); } @@ -122,6 +132,22 @@ public QwpBufferWriter getBuffer() { return buffer; } + /** + * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...}, + * excluding the two section varints) that {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesLen() { + return deltaEntriesEnd - deltaEntriesStart; + } + + /** + * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY + * region {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesStart() { + return deltaEntriesStart; + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4a499300..942af99b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3681,24 +3681,41 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write. + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the + // frame is published. // // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: - // appendSymbols advances pd.size() only after a full write, whereas + // the persist advances pd.size() only after a full write, whereas // sentMaxSymbolId only advances after the WHOLE frame is published (via // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist - // threw (a short write -- disk full/quota), the frame was not published - // and sentMaxSymbolId stayed put, while the symbols before the failing - // one are already on disk. Keying the resume point off sentMaxSymbolId+1 - // would then re-append that persisted prefix on the retry, duplicating - // entries and corrupting the dense id->symbol mapping recovery relies on - // (position i must be symbol id i). pd.size() resumes exactly past what is - // already durable (the write overwrites any torn trailing bytes at - // appendOffset), so the write-ahead is idempotent under retry. In the - // happy path pd.size() equals sentMaxSymbolId+1, so the range -- and the - // behaviour -- are unchanged; the single positioned write just replaces - // the previous one-syscall-per-new-symbol loop. - pd.appendSymbols(globalSymbolDictionary, pd.size(), currentBatchMaxSymbolId); + // threw (short write -- disk full/quota) or the publish threw, the frame + // was not published and sentMaxSymbolId stayed put, while the symbols + // before the failure are already on disk. Keying the resume point off + // sentMaxSymbolId+1 would re-append that persisted prefix on the retry, + // duplicating entries and corrupting the dense id->symbol mapping recovery + // relies on (position i must be symbol id i). pd.size() resumes exactly + // past what is already durable, so the write-ahead is idempotent. + int from = pd.size(); + if (currentBatchMaxSymbolId < from) { + return; // nothing new to persist (warm batch, or an idempotent retry) + } + // Fast path: the frame the encoder just built already holds these symbols + // in its delta section as [len][utf8]... -- byte-identical to what + // PersistedSymbolDict stores. In the common case pd.size() equals the + // frame's delta start id (sentMaxSymbolId+1), so persist those bytes + // straight from the frame instead of re-encoding the symbols. After a + // failed publish the durable size has run ahead of the wire baseline, so + // the frame's delta covers MORE than remains to persist; then re-encode + // just the [from .. currentBatchMaxSymbolId] suffix. + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } } private void resetSymbolDictStateForNewConnection() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index f5f21207..6b9eee60 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2192,6 +2192,15 @@ private int sendDictCatchUp() { // Latch a terminal (the data must be resent after the cap is // raised) rather than calling fail() -- which, from inside the // catch-up, would re-enter connectLoop (see CatchUpSendException). + // + // Tradeoff (heterogeneous / rolling-cap clusters): a symbol + // accepted under a larger/absent cap can hit this on failover to a + // smaller-cap node, and the hard terminal does NOT self-recover if + // a later node advertises a larger cap -- the producer must be + // resumed after the data is resent (or the cap raised). Bounding + // symbol size at ingest, or a settle budget across reconnects + // before latching, would relax this, but both are larger changes; + // the terminal keeps the homogeneous common case livelock-free. LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index ca769181..55c04ba7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -148,6 +148,30 @@ public static void removeOrphan(String slotDir) { Files.remove(slotDir + "/" + FILE_NAME); } + /** + * Appends {@code count} already-encoded entries -- {@code [len varint][utf8]...}, + * the exact layout {@link #appendSymbols} produces -- verbatim from {@code addr} + * in a single write. The producer uses this when it already holds those bytes + * (the symbol-dict delta section the frame encoder just wrote), so it does not + * re-encode the same symbols. Writes {@code len} bytes and advances {@code size} + * by {@code count}. Same durability/idempotency contract as {@link #appendSymbols}: + * no fsync, and a short write throws WITHOUT advancing {@code size}/{@code + * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range + * at the same offset. No-op when the range is empty or the dictionary is closed. + */ + public void appendRawEntries(long addr, int len, int count) { + if (closed || count <= 0 || len <= 0) { + return; + } + long written = Files.write(fd, addr, len, appendOffset); + if (written != len) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + len + ", actual=" + written + ']'); + } + appendOffset += len; + size += count; + } + /** * Appends one symbol, extending the on-disk dictionary. The caller appends a * frame's new symbols BEFORE publishing that frame, so the write ordering diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index d580f886..67fd7438 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -91,6 +91,62 @@ public void testAppendPersistsAcrossReopen() throws Exception { }); } + @Test + public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { + // M1: the producer persists the frame's already-encoded delta bytes via + // appendRawEntries instead of re-encoding the symbols. Those bytes are the + // same [len][utf8]... layout appendSymbols writes, so both must produce an + // identical, recoverable dictionary. Encode a range with appendSymbols, + // reopen to grab its on-disk entry bytes, replay them through + // appendRawEntries into a fresh dict, and assert the recovered symbols + // match -- including an empty entry mid-range. + assertMemoryLeak(() -> { + Path src = Files.createTempDirectory("qwp-symdict-src"); + Path dst = Files.createTempDirectory("qwp-symdict-dst"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + + PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + encoded.appendSymbols(dict, 0, 2); + encoded.close(); + + // Reopen to obtain the on-disk entry region [len][utf8]... verbatim, + // then replay it byte-for-byte into a fresh dict via appendRawEntries. + PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + try { + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); + try { + raw.appendRawEntries(reopened.loadedEntriesAddr(), + reopened.loadedEntriesLen(), reopened.size()); + Assert.assertEquals(3, raw.size()); + } finally { + raw.close(); + } + } finally { + reopened.close(); + } + + // The raw-appended dict must recover the same dense symbols. + PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString()); + try { + Assert.assertEquals(3, recovered.size()); + ObjList symbols = recovered.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + } finally { + recovered.close(); + } + } finally { + rmDir(src); + rmDir(dst); + } + }); + } + @Test public void testAppendSymbolsBatchWritesDenseRange() throws Exception { // appendSymbols persists a whole id range in one write (the hot-path From 10a125c05d169dcb87dc38eeeab73d28621add70 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 11:10:00 +0100 Subject: [PATCH 24/78] Guard catch-up frame overflow, add biting tests Two defensive spots in the delta symbol-dict path had no biting test, and one guard was incomplete (M3). Complete the catch-up frame-size guard (M3b). The int-overflow hardening covered ensureSentDictCapacity (the mirror growth) but not the single catch-up frame: with no server cap, frameLen = HEADER + varints + symbolsLen is an int and would wrap negative as symbolsLen approaches the mirror ceiling, feeding a bad Unsafe.malloc. sendDictCatchUp's budget already keeps each chunk under that bound, so this is unreachable at real cardinality -- but the guard must be local so a future caller cannot overflow it silently. Compute the size in long and fail loud (CatchUpSendException) before the malloc, matching the mirror-side guard. Cover accumulateSentDict's partial-overlap tail (M3a). A delta that straddles the mirror tip (deltaStart < sentDictCount < deltaEnd) must copy only the new tail, not drop the whole frame. The monotonic producer never emits a straddling delta in steady state, so reverting to the pre-fix drop-whole-frame guard passed every test; it is reachable on a torn-dict replay (mirror seeded smaller than a frame's coverage), where dropping the tail would leave the reconnect catch-up incomplete and shift server ids. Add a white-box test that drives the straddle directly. Both new tests fail with their production line reverted (the mirror stays at 1 id; the frame guard falls through to a negative malloc that throws IllegalArgumentException, not the clean terminal) and pass with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 18 ++- ...WebSocketSendLoopCatchUpAlignmentTest.java | 132 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6b9eee60..6d8ed64b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2234,10 +2234,24 @@ private int sendDictCatchUp() { * the caller turns it into a single, non-re-entrant reconnect. */ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { - int payloadLen = NativeBufferWriter.varintSize(deltaStart) + // Compute the frame size in long and fail loud if it would overflow the int + // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each + // chunk's symbol bytes under the budget, so this is unreachable at real + // cardinality -- but the mirror-side ensureSentDictCapacity guards the same + // math, and a future caller must not be able to overflow this one silently. + long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart) + NativeBufferWriter.varintSize(deltaCount) + symbolsLen; - int frameLen = QwpConstants.HEADER_SIZE + payloadLen; + long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL; + if (frameLenL > MAX_SENT_DICT_BYTES) { + LineSenderException err = new LineSenderException( + "symbol dictionary catch-up frame exceeds the maximum size [" + + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + int payloadLen = (int) payloadLenL; + int frameLen = (int) frameLenL; long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); try { Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 9b2195d9..79640f25 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -43,8 +43,12 @@ import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -211,6 +215,76 @@ public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Excep }); } + @Test + public void testAccumulateSentDictPartialOverlapExtendsMirror() throws Exception { + // M3: accumulateSentDict must handle a delta that STRADDLES the mirror tip + // (deltaStart < sentDictCount < deltaStart+deltaCount) by copying only the + // new tail, not dropping the whole frame. The monotonic producer never emits + // a straddling delta in steady state (so the pre-fix drop-whole-frame guard + // passed every test), but a torn-dict replay can seed the mirror smaller than + // a frame's coverage. Seed the mirror with 1 symbol, feed a [0..2] delta, and + // assert the mirror extends to all 3 -- pre-fix it stayed at 1, leaving the + // reconnect catch-up incomplete and shifting server-side ids. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "aa"); // sentDictCount = 1, mirror holds "aa" + int[] frameLen = new int[1]; + long frame = buildDeltaFrame(0, new String[]{"aa", "bb", "cc"}, frameLen); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "accumulateSentDict", long.class, int.class, int.class); + m.setAccessible(true); + m.invoke(loop, frame, frameLen[0], 0); + } finally { + Unsafe.free(frame, frameLen[0], MemoryTag.NATIVE_DEFAULT); + } + assertEquals("straddling delta must extend the mirror to all 3 ids", + 3, readInt(loop, "sentDictCount")); + assertEquals("mirror must hold the two new tail symbols after the " + + "already-held prefix, gap-free", + Arrays.asList("aa", "bb", "cc"), readMirrorSymbols(loop)); + } finally { + loop.close(); + } + } + }); + } + + @Test + public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { + // M3: sendDictCatchUp caps each chunk under the budget, so the single-frame + // catch-up path cannot overflow its int frameLen at any real cardinality. The + // guard must still be LOCAL -- a future caller must not be able to feed a + // wrapped-negative frameLen to Unsafe.malloc. An oversized symbolsLen must + // fail loud (CatchUpSendException) BEFORE the malloc; the guard fires before + // symbolsAddr is read, so a dummy address is fine. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "sendCatchUpChunk", int.class, int.class, long.class, int.class); + m.setAccessible(true); + // symbolsLen past the mirror ceiling: HEADER + varints + symbolsLen + // overflows an int, so the guard must reject it before malloc. + m.invoke(loop, 0, 1, 0L, Integer.MAX_VALUE - 4); + fail("an overflowing catch-up frame size must fail loud, not malloc negative"); + } catch (InvocationTargetException e) { + assertEquals("overflow must surface as CatchUpSendException", + "CatchUpSendException", e.getCause().getClass().getSimpleName()); + assertTrue("message must name the frame-size guard: " + e.getCause().getMessage(), + e.getCause().getMessage().contains("catch-up frame exceeds the maximum size")); + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { @@ -226,6 +300,64 @@ private static void appendFrames(CursorSendEngine engine, int count) { } } + // Builds a QWP delta frame [12-byte header][deltaStart varint][deltaCount + // varint][ [len varint][utf8] ... ] for the given symbols. accumulateSentDict + // skips the header, so its content is irrelevant; the caller frees the frame. + private static long buildDeltaFrame(int deltaStart, String[] symbols, int[] outLen) { + int deltaCount = symbols.length; + int size = 12 + varintSize(deltaStart) + varintSize(deltaCount); + for (String s : symbols) { + size += varintSize(s.getBytes(StandardCharsets.UTF_8).length) + + s.getBytes(StandardCharsets.UTF_8).length; + } + long addr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + long p = writeVarint(addr + 12, deltaStart); + p = writeVarint(p, deltaCount); + for (String s : symbols) { + byte[] b = s.getBytes(StandardCharsets.UTF_8); + p = writeVarint(p, b.length); + for (byte x : b) { + Unsafe.getUnsafe().putByte(p++, x); + } + } + outLen[0] = size; + return addr; + } + + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getInt(loop); + } + + // Parses the loop's native sent-dictionary mirror ([len varint][utf8]...) back + // into the symbol strings a reconnect catch-up would re-register. + private static List readMirrorSymbols(CursorWebSocketSendLoop loop) throws Exception { + long addr = readLong(loop, "sentDictBytesAddr"); + int len = readInt(loop, "sentDictBytesLen"); + List out = new ArrayList<>(); + long p = addr; + long limit = addr + len; + while (p < limit) { + long l = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + l |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + byte[] bytes = new byte[(int) l]; + for (int i = 0; i < l; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(p++); + } + out.add(new String(bytes, StandardCharsets.UTF_8)); + } + return out; + } + // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response // handler, mimicking the server acking a catch-up frame (which carries no tables). private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception { From fa50e81f0288077d2e0040314fb77a0ce5a780b7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 12:26:08 +0100 Subject: [PATCH 25/78] Fix orphan-drain losing symbol-dict mirror seed The send loop seeded its reconnect catch-up mirror by calling PersistedSymbolDict.takeLoadedEntries(), a one-shot ownership transfer that zeroed the dictionary's loaded-entries buffer. The foreground sender builds a single loop, so that was safe. But BackgroundDrainer builds a fresh send loop per wire session against the SAME, persistent engine when a durable-ack capability gap forces a mid-drain recycle. The second loop then found an empty loaded-entries buffer, seeded an empty mirror (sentDictCount = 0), sent no reconnect catch-up, and the first replayed delta frame (deltaStart > 0) tripped the torn-dict guard -- falsely quarantining a healthy slot with a bogus "resend required" terminal and defeating the capability-gap settle-retry for delta slots. The send loop now COPIES the loaded entries into its own mirror and leaves the dictionary owning its buffer for the engine's lifetime, so every recycled loop re-seeds. PersistedSymbolDict.close() frees the dictionary's copy; each loop frees its own copy on exit. Peak footprint is unchanged (the drainer closes loop N before building loop N+1). Removes the now-unused takeLoadedEntries() and loadedEntriesTaken, and adds a regression test that builds two loops against one recovered engine and asserts the second re-seeds (pre-fix its mirror was empty). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 24 ++++--- .../client/sf/cursor/PersistedSymbolDict.java | 33 ++------- ...CursorWebSocketSendLoopMirrorLeakTest.java | 70 +++++++++++++++++++ 3 files changed, 91 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6d8ed64b..d910a6ca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -543,15 +543,21 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, if (pd != null && pd.size() > 0) { int len = pd.loadedEntriesLen(); if (len > 0) { - // Adopt the persisted dictionary's loaded-entries buffer as the - // mirror's initial backing instead of copying it and leaving the - // dictionary to retain a second copy for the engine's lifetime. - // The producer's readLoadedSymbols() -- the only other consumer -- - // runs earlier in setCursorEngine; the drainer path has no - // producer consumer. takeLoadedEntries() transfers ownership, so - // this loop's mirror lifecycle frees it (not pd.close()). The - // buffer's allocated size equals loadedEntriesLen. - sentDictBytesAddr = pd.takeLoadedEntries(); + // COPY the persisted dictionary's loaded-entries buffer into this + // loop's own mirror rather than taking ownership of it. The engine + // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan + // drainer path: BackgroundDrainer builds a fresh send loop per wire + // session against the same engine on a durable-ack capability-gap + // recycle. A one-shot ownership transfer would leave every loop + // after the first with an EMPTY mirror -- it would then send no + // reconnect catch-up, and the first replayed delta frame + // (deltaStart > 0) would trip the torn-dict guard, falsely + // quarantining a healthy slot. Copying keeps the dictionary's + // loaded entries intact for the engine's lifetime so every + // recycled loop re-seeds; pd.close() (at engine close) frees the + // dictionary's copy, this loop frees its own copy on exit. + sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); sentDictBytesCapacity = len; sentDictBytesLen = len; // Set the count only alongside the bytes so sentDictCount can diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 55c04ba7..bf5fc39f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -95,15 +95,14 @@ public final class PersistedSymbolDict implements QuietCloseable { private boolean closed; // In-memory copy of the entry region [len][utf8]... exactly as on disk, // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. Consumed once to seed - // the producer's id map (readLoadedSymbols) and then ADOPTED by the send loop's - // catch-up mirror (takeLoadedEntries), which takes ownership so this file no - // longer retains a second copy of the dictionary for the engine's lifetime. + // orphan-drain). Zero/empty for a freshly created file. READ (not consumed) to + // seed the producer's id map (readLoadedSymbols) and to seed the send loop's + // catch-up mirror, which COPIES it. This file retains ownership for the engine's + // lifetime -- the orphan drainer builds a fresh send loop per wire session + // against the same engine, and each must re-seed its mirror -- and frees this + // buffer in close(). private long loadedEntriesAddr; private int loadedEntriesLen; - // True once takeLoadedEntries() handed loadedEntriesAddr to the send-loop - // mirror; guards a stale readLoadedSymbols() call (which must run first). - private boolean loadedEntriesTaken; private long scratchAddr; private int scratchCap; private int size; @@ -282,7 +281,6 @@ public int loadedEntriesLen() { * recovered. */ public ObjList readLoadedSymbols() { - assert !loadedEntriesTaken : "readLoadedSymbols() must run before takeLoadedEntries()"; ObjList out = new ObjList<>(Math.max(size, 1)); long p = loadedEntriesAddr; long limit = p + loadedEntriesLen; @@ -313,25 +311,6 @@ public int size() { return size; } - /** - * Transfers ownership of the loaded-entries buffer to the caller and returns - * its base address (0 when nothing was recovered). The send loop adopts it as - * the initial backing of its catch-up mirror rather than copying it, so this - * file stops retaining a second copy of the dictionary for the engine's - * lifetime. After this call {@link #close()} no longer frees the buffer -- the - * caller owns it. The producer's {@link #readLoadedSymbols()} is the only other - * consumer and MUST run first: {@code setCursorEngine} seeds the producer - * before the send loop is constructed, and the drainer path has no producer - * consumer. The buffer was allocated with {@link MemoryTag#NATIVE_DEFAULT}. - */ - public long takeLoadedEntries() { - long addr = loadedEntriesAddr; - loadedEntriesAddr = 0L; - loadedEntriesLen = 0; - loadedEntriesTaken = true; - return addr; - } - private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 19626ba4..7869882d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -33,6 +33,7 @@ import org.junit.Test; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.util.Comparator; @@ -96,6 +97,69 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { } } + @Test + public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { + // C1 regression: the orphan drainer (BackgroundDrainer) builds a NEW + // CursorWebSocketSendLoop per wire session against the SAME, persistent + // engine when a durable-ack capability gap forces a mid-drain recycle. The + // recovery mirror seed must survive that recycle. If the first loop CONSUMED + // the persisted dictionary's loaded entries (a one-shot ownership transfer), + // the second loop seeds an EMPTY mirror (sentDictCount = 0), sends no + // reconnect catch-up, and the first replayed delta frame (deltaStart > 0) + // trips the torn-dict guard -- falsely quarantining a healthy slot with a + // bogus "resend required" terminal. Copying the entries (leaving the + // dictionary intact for the engine's lifetime) lets every recycled loop + // re-seed. Pre-fix, loop2's sentDictCount is 0 and this assertion fails. + Path sfDir = Files.createTempDirectory("qwp-mirror-reseed"); + try { + populateRecoverableSlot(sfDir); + Path slot = sfDir.resolve("default"); + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull(pd); + int dictSize = pd.size(); + Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0); + + // Session 1 seeds its mirror from the persisted dictionary. + CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine); + try { + Assert.assertEquals("session-1 mirror must seed from the persisted dict", + dictSize, readInt(loop1, "sentDictCount")); + } finally { + loop1.close(); + } + + // Session 2 against the SAME engine (the drainer recycle): the + // seed must NOT have been consumed -- the mirror must re-seed to + // the full dictionary so the reconnect catch-up is complete. + CursorWebSocketSendLoop loop2 = newRecoveryLoop(engine); + try { + Assert.assertEquals("recycled session-2 mirror must re-seed from the " + + "persisted dict (pre-fix it was 0)", + dictSize, readInt(loop2, "sentDictCount")); + } finally { + loop2.close(); + } + } + }); + } finally { + rmDir(sfDir); + } + } + + // Constructs a recovery send loop but does NOT start it: the ctor seeds the + // catch-up mirror synchronously, which is all these tests observe. The + // reconnect factory is never invoked. + private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine) { + return new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + } + private static void populateRecoverableSlot(Path sfDir) throws Exception { try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); @@ -117,6 +181,12 @@ private static void populateRecoverableSlot(Path sfDir) throws Exception { } } + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getInt(loop); + } + private static void rmDir(Path dir) throws IOException { if (dir == null || !Files.exists(dir)) { return; From 444199ea881da44aa9ee988df1b220ff5b568d20 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 12:32:53 +0100 Subject: [PATCH 26/78] Fix symbol-dict durability doc; add recovery test The PersistedSymbolDict class doc claimed a host-crash-torn dictionary is "caught at replay by the send loop's guard ... rather than corrupting the target table." The guard is tail-only: it fires only when a frame's delta start id exceeds the recovered dictionary size. It does NOT catch an interior page lost out of order (reads back as zeroes that parse as empty-string entries) or a failed best-effort truncate that leaves a stale trailing entry -- either shifts the dense id->symbol mapping without changing the entry count the guard checks, so a replay silently misattributes symbols. Correct the class doc and the truncate comment to describe the actual, tail-only protection and name the residual host-crash exposure (within the already-documented "not host-crash durable" boundary); note that a per-entry or running CRC would close it. Add testRecoveryAfterFailedPublishReplaysGapFree: a failed publish persists a frame's symbol without recording the frame, leaving the persisted dictionary a strict superset of the recorded frames. The test recovers the slot on a fresh sender against a dictionary-reconstructing server and asserts the catch-up re-registers the whole superset (the unrecorded symbol included) and the replay is gap-free -- the end-to-end fail -> recover -> replay chain the existing no-duplicate test omitted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 23 ++++- .../qwp/client/DeltaDictRecoveryTest.java | 94 +++++++++++++++++++ 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index bf5fc39f..986039c2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -67,9 +67,18 @@ * dictionary is a superset of every recoverable frame's references. It is NOT * sufficient for a host/power crash, where unflushed pages can be lost out * of order and the dictionary may end up torn relative to the frames it serves -- - * exactly as the segment frames themselves may be lost on a host crash. A torn - * dictionary is caught at replay by the send loop's guard, which fails loudly - * (the unreplayable data must be resent) rather than corrupting the target table. + * exactly as the segment frames themselves may be lost on a host crash. The send + * loop's replay guard catches the COMMON host-crash outcome -- a truncated tail, + * where a frame's delta start id exceeds the recovered dictionary size -- and + * fails loudly (the unreplayable data must be resent) rather than sending a + * gapped frame. It does NOT catch every host-crash tear: an interior page lost + * out of order reads back as zeroes that parse as empty-string entries, and a + * failed best-effort truncate (see {@link #open}) can leave a stale trailing + * entry -- either SHIFTS the dense id->symbol mapping without changing the entry + * count the guard checks, so a replay silently misattributes symbols. That + * residual exposure sits within the "not host-crash durable" boundary above; a + * per-entry or running CRC would be needed to close it (the 3 reserved header + * bytes leave room for a future integrity field). *

* A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. @@ -367,8 +376,12 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { // shift every subsequent dense id. Without this, appendOffset already // lands just past the last complete entry, so the next append // overwrites FROM there, but only truncation guarantees nothing - // survives BEYOND it. Best-effort: a failed truncate just forgoes the - // hardening and falls back to the overwrite-from-appendOffset behaviour. + // survives BEYOND it. Best-effort: a failed truncate (rare -- an I/O + // error) leaves a latent silent-corruption path, not merely a lost + // optimization: if a later, shorter append then leaves stale bytes past + // its end, a subsequent recovery mis-parses them as a ghost symbol and + // shifts the dense ids, and the count-based replay guard does not catch + // it (see the class-level durability note). long validLen = HEADER_SIZE + entriesLen; if (validLen < len) { Files.truncate(fd, validLen); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 8174ed24..222eeebe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -480,6 +480,100 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception }); } + @Test + public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { + // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay. + // A failed publish persists the frame's symbol (write-ahead) but does NOT + // record the frame, so the persisted dictionary becomes a strict SUPERSET of + // the recorded frames' references. A recovering sender must still replay + // gap-free: it re-registers the whole (superset) dictionary via the catch-up + // -- including the symbol whose frame never reached disk -- so the fresh + // server reconstructs a complete, gap-free dictionary. The sibling + // testFailedPublishDoesNotDuplicatePersistedSymbols proves the dict has no + // duplicate after the failed publish; this proves the resulting slot then + // recovers and replays end-to-end against a real server. + assertMemoryLeak(() -> { + // Phase 1: a silent server (no acks) + a small SF segment. Four small + // rows register sym-0..sym-3 and their frames are recorded; a fifth, + // oversized row registers (persists) sym-4 but its frame is too large for + // the segment, so appendBlocking throws and the frame is NOT recorded. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + Sender s1 = Sender.fromConfig(cfg); + try { + for (int i = 0; i < 4; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + // Oversized frame: sym-4 is persisted (write-ahead) before the + // publish fails, so the dictionary runs one ahead of the frames. + s1.table("m").symbol("s", "sym-4") + .stringColumn("p", TestUtils.repeat("x", 8000)) + .longColumn("v", 4).atNow(); + try { + s1.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + // PAYLOAD_TOO_LARGE -- frame not recorded, sym-4 stays persisted + } + } finally { + try { + // Re-flushes the still-buffered oversized row and fails again + // (expected); resources are still released, and the idempotent + // write-ahead does not re-append sym-4. + s1.close(); + } catch (LineSenderException ignored) { + } + } + } + + // The persisted dictionary must hold the superset: sym-0..sym-4 (5 ids), + // one more than the four recorded frames reference, with no duplicate. + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("failed publish must leave the dict a superset (sym-0..sym-4)", + 5, pd.size()); + } finally { + pd.close(); + } + + // Phase 2: recover against a fresh server that reconstructs its + // dictionary from the wire. The recovering sender must re-register all 5 + // symbols via a catch-up (sym-4 exists ONLY in the dictionary -- no frame + // carries it) and replay the 4 recorded frames, leaving a complete, + // gap-free server dictionary. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 5) { + Thread.sleep(20); + } + } + Assert.assertTrue("recovery must send a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + List dict = handler.dictSnapshot(); + Assert.assertEquals("recovered dictionary must include the failed-publish symbol", + 5, dict.size()); + for (int i = 0; i < 5; i++) { + Assert.assertEquals("dictionary id " + i + " must be gap-free", + "sym-" + i, dict.get(i)); + } + } + }); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); From 7a9565397c18aaa5411434fd8a27f0f49317b416 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 13:48:20 +0100 Subject: [PATCH 27/78] Discard a surviving symbol dict on fresh start A fresh store-and-forward slot inherited a stale .symbol-dict when the best-effort removeOrphan failed to delete it (e.g. a Windows share lock) or a crash landed in the close window: open() parses and TRUSTS any survivor. Because the fresh-start producer is not seeded from the dictionary (that is gated on wasRecoveredFromDisk, while the send-loop mirror and the persist resume point key off pd.size()), the producer's ids then diverged from the dictionary the send loop replays, silently misattributing symbol values after a reconnect. The dictionary is load-bearing, unlike the max()-clamped ack watermark whose removeOrphan+open pattern it had copied. Enforce "fresh start -> empty dictionary" at the source: openClean() truncates any survivor via openCleanRW instead of trusting it, and if the clean open itself fails the sender falls back to full self-sufficient frames, which is also safe. The recovery path keeps open(); removeOrphan stays for the fully-drained close. Both regression tests fail without the fix (dict size 2, not 0): - PersistedSymbolDictTest.testOpenCleanDiscardsSurvivingDictionary - EmptyOrphanSlotChurnTest.testFreshStartDiscardsSurvivingStaleDictionary Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 17 +++++-- .../client/sf/cursor/PersistedSymbolDict.java | 29 +++++++++-- .../sf/cursor/EmptyOrphanSlotChurnTest.java | 44 +++++++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 49 +++++++++++++++++++ 4 files changed, 131 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 3bc0a185..4b692288 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -321,10 +321,19 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); - // Same stale-side-file hygiene for the symbol dictionary: a - // fresh slot starts with an empty dictionary. - PersistedSymbolDict.removeOrphan(sfDir); - persistedDictInProgress = PersistedSymbolDict.open(sfDir); + // A fresh slot MUST start with an EMPTY symbol dictionary. + // Unlike the ack watermark above -- a discardable optimization a + // max() clamp protects -- the dictionary is load-bearing: a + // delta frame referencing an id missing from it is unrecoverable, + // and a STALE dictionary inherited here (the segments are gone, so + // the producer is NOT seeded from it) shifts the dense id->symbol + // mapping and silently misattributes symbols on the next + // reconnect. openClean() truncates any survivor to empty rather + // than trusting a best-effort delete that may have failed (e.g. a + // Windows share lock); if the clean open itself fails, + // persistedSymbolDict stays null and the sender falls back to full + // self-sufficient frames, which is also safe. + persistedDictInProgress = PersistedSymbolDict.openClean(sfDir); } MmapSegment initial; String initialPath = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 986039c2..3fc76fd4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -147,10 +147,31 @@ public static PersistedSymbolDict open(String slotDir) { } /** - * Best-effort removal of a stale dictionary file. Used at fresh-start (a - * stale dict with no segments behind it is meaningless) and at fully-drained - * close (the slot is empty, nothing references the dictionary any more), - * mirroring {@link AckWatermark#removeOrphan}. + * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding + * any surviving content. This is the fresh-start counterpart to {@link #open}: + * a slot with no recovered segments must start with an empty dictionary, so a + * dictionary left by a prior lifecycle -- a fully-drained slot whose + * best-effort delete failed, or a crash in the close window -- must NOT be + * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for + * recovery/orphan-drain replay, this truncates it: the fresh-start producer is + * not seeded from the dictionary, so trusting a survivor would leave the + * producer's ids diverged from the dictionary the send loop replays and + * silently misattribute symbols on the next reconnect. Truncating (rather than + * relying on an unlink succeeding first) closes the gap even when the delete is + * refused -- e.g. a Windows share lock. Returns {@code null} on I/O failure, so + * the caller falls back to full self-sufficient frames exactly as {@link #open} + * does. + */ + public static PersistedSymbolDict openClean(String slotDir) { + return openFresh(slotDir + "/" + FILE_NAME); + } + + /** + * Best-effort removal of a stale dictionary file. Used at fully-drained close + * (the slot is empty, nothing references the dictionary any more), mirroring + * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT + * use this -- it opens a clean dictionary via {@link #openClean} instead, so a + * failed delete cannot leave a stale dictionary a new session would trust. */ public static void removeOrphan(String slotDir) { Files.remove(slotDir + "/" + FILE_NAME); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java index 5c7607a6..2efee2e4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.Files; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -35,6 +36,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; /** * Regression test for M6 — drainer adopting an empty orphan slot would @@ -87,6 +89,48 @@ public void tearDown() { Files.remove(sfDir); } + @Test + public void testFreshStartDiscardsSurvivingStaleDictionary() throws Exception { + // Regression: a prior fully-drained lifecycle can leave a stale + // .symbol-dict behind (a best-effort delete that failed, or a crash in the + // close window) with NO segments. A fresh start must DISCARD it -- the + // dictionary is load-bearing and the fresh-start producer is not seeded + // from it, so trusting a survivor would diverge the producer ids from the + // dictionary the send loop replays and misattribute symbols on reconnect. + TestUtils.assertMemoryLeak(() -> { + // Pre-seed a stale dictionary in the slot, with no segments behind it. + PersistedSymbolDict stale = PersistedSymbolDict.open(sfDir); + assertNotNull(stale); + try { + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + assertEquals(2, stale.size()); + } finally { + stale.close(); + } + + // A fresh start (no recovered segments) must open a CLEAN, empty + // dictionary -- not inherit the survivor. + try (CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024)) { + assertFalse("fresh start must not report a disk recovery", + engine.wasRecoveredFromDisk()); + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + assertNotNull(pd); + assertEquals("fresh start must discard the surviving stale dictionary", + 0, pd.size()); + } + + // The survivor's bytes are physically gone, not just hidden. + PersistedSymbolDict reopened = PersistedSymbolDict.open(sfDir); + assertNotNull(reopened); + try { + assertEquals(0, reopened.size()); + } finally { + reopened.close(); + } + }); + } + @Test public void testNeverPublishedCloseLeavesNoSfaFiles() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 67fd7438..cc45c23b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -299,6 +299,55 @@ public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { }); } + @Test + public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { + // A fresh start must NOT inherit a dictionary left by a prior lifecycle: + // openClean() truncates any survivor to empty, where open() would recover + // (and TRUST) it. Trusting a survivor whose segments are gone -- the + // fresh-start producer is not seeded from it -- shifts the dense id->symbol + // mapping and misattributes symbols on the next reconnect. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict-clean"); + try { + PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(stale); + try { + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + Assert.assertEquals(2, stale.size()); + } finally { + stale.close(); + } + + // Fresh start: openClean yields an EMPTY dictionary regardless of + // the survivor, and appends from id 0 again. + PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString()); + Assert.assertNotNull(fresh); + try { + Assert.assertEquals(0, fresh.size()); + Assert.assertEquals(0, fresh.readLoadedSymbols().size()); + fresh.appendSymbol("freshA"); + Assert.assertEquals(1, fresh.size()); + } finally { + fresh.close(); + } + + // The survivor's bytes are physically gone, not just hidden: a + // subsequent recovery open() sees only the post-clean content. + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(reopened); + try { + Assert.assertEquals(1, reopened.size()); + Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0)); + } finally { + reopened.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { From 6d14727421438637ed9e3be5391d34a19e94c567 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 14:05:04 +0100 Subject: [PATCH 28/78] Harden delta symbol-dict send and persist paths Four follow-ups from the delta symbol-dictionary review. Catch-up NACK no longer launders the poison detector. After a reconnect the loop ships the head data frame before tryReceiveAcks reads the server's NACK of a dictionary catch-up frame, so handleServerRejection saw dataFrameSentThisConnection=true and attributed the NACK a wire seq below the replay head -- fsn = fsnAtZero+cappedSeq, negative when replayStart < catchUpFrames. recordHeadRejectionStrike then set poisonFsn to that value (often -1, the "no suspect" sentinel), wiping a genuine in-progress poison run and reporting a bogus fsn. Route an fsn <= ackedFsn rejection to the pre-send path (surface + recycle, no strike, no watermark advance), symmetric with the success path's engine.acknowledge() no-op below ackedFsn. A real replayed data frame is at fsn > ackedFsn, so it is never caught. Regression test: testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState. Persist short-write now surfaces as LineSenderException. A disk-full during the write-ahead persist threw a raw IllegalStateException that escaped flush() unwrapped, unlike every other flush-path failure (e.g. the cursor append in sealAndSwapBuffer). Wrap it so a caller catching LineSenderException around flush() sees a persist disk-full too; a JVM Error still propagates. PersistedSymbolDict close() and the append methods are synchronized. close() is callable from any thread (a shutdown hook); without mutual exclusion a close racing an in-flight append could free the scratch buffer or close the fd mid-write and let the write land on a descriptor the OS has reused for another file (silent cross-file corruption). The appendSymbols re-encode branch now has a regression test. After a failed publish leaves the durable dictionary size ahead of the wire baseline, a later flush introducing a new symbol re-encodes only the [pd.size() .. currentBatchMaxSymbolId] suffix; keying that off pd.size() (not sentMaxSymbolId+1) keeps it idempotent. Regression test: testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating. Both new tests fail with their production hunk reverted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 32 +++++++--- .../sf/cursor/CursorWebSocketSendLoop.java | 19 ++++++ .../client/sf/cursor/PersistedSymbolDict.java | 19 ++++-- .../qwp/client/DeltaDictRecoveryTest.java | 64 +++++++++++++++++++ ...ursorWebSocketSendLoopPoisonFrameTest.java | 62 ++++++++++++++++++ 5 files changed, 181 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 942af99b..bb1256dd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3707,14 +3707,30 @@ private void persistNewSymbolsBeforePublish() { // failed publish the durable size has run ahead of the wire baseline, so // the frame's delta covers MORE than remains to persist; then re-encode // just the [from .. currentBatchMaxSymbolId] suffix. - if (from == sentMaxSymbolId + 1) { - QwpBufferWriter buffer = encoder.getBuffer(); - pd.appendRawEntries( - buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), - encoder.getDeltaEntriesLen(), - currentBatchMaxSymbolId - from + 1); - } else { - pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + try { + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } + } catch (Throwable t) { + // A short write (disk full / quota) to the persisted dictionary throws + // a low-level IllegalStateException. Surface it as a LineSenderException + // -- like every other flush-path failure, e.g. the cursor append in + // sealAndSwapBuffer -- so a caller catching LineSenderException around + // flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the short + // write, so the still-buffered rows re-persist the same range + // idempotently on retry. A JVM Error is never a persist failure; let it + // propagate. + if (t instanceof Error) { + throw (Error) t; + } + throw new LineSenderException("failed to persist symbol dictionary before publish", t); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index d910a6ca..478d33c5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2811,6 +2811,25 @@ private void handleServerRejection(long wireSeq) { wireSeq, highestSent); } long fsn = fsnAtZero + cappedSeq; + if (fsn <= engine.ackedFsn()) { + // The clamped wire seq maps at or below the replay head, so this + // NACK is for a dictionary catch-up frame -- which occupies the + // already-acked wire sequences below replayStart -- not a data frame + // this connection sent. (dataFrameSentThisConnection can be true here + // because trySendOne ships the head data frame before tryReceiveAcks + // reads the catch-up's NACK in the same loop iteration.) Attributing + // it a data FSN would key recordHeadRejectionStrike() off a + // below-baseline FSN -- negative when replayStart < catchUpFrames -- + // colliding with the poisonFsn == -1 "no suspect" sentinel and + // laundering a genuine poison run, and would report a bogus FSN. + // Treat it exactly like a pre-send rejection: surface + recycle, no + // poison strike, no watermark advance. Symmetric with the success + // path, where engine.acknowledge() no-ops at or below ackedFsn. A + // real replayed data frame is at fsn > ackedFsn, so it is never + // caught here. + handlePreSendRejection(wireSeq, status, category, policy); + return; + } // Best-effort table attribution: the parser populates // response.tableNames on error frames the same way it does on // STATUS_OK. If exactly one table was named, surface it; if diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 3fc76fd4..d2651923 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -83,9 +83,14 @@ * A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. *

- * Lifecycle: single-writer (the producer / user thread). Read once at - * {@link #open} to seed in-memory state on recovery or orphan-drain. Owner - * (the engine) closes it. Not thread-safe for concurrent writers. + * Lifecycle: single-writer (the producer / user thread) for appends. Read + * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The + * owner (the engine) closes it, and {@code close()} is callable from any thread + * (a shutdown hook, test cleanup). {@code close()} and the append methods are + * therefore {@code synchronized}: without that, a close racing an in-flight append + * could free the scratch buffer or close the fd mid-write and let the write land + * on a descriptor the OS has reused for another file (silent cross-file + * corruption). Not thread-safe for concurrent writers. */ public final class PersistedSymbolDict implements QuietCloseable { @@ -188,7 +193,7 @@ public static void removeOrphan(String slotDir) { * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range * at the same offset. No-op when the range is empty or the dictionary is closed. */ - public void appendRawEntries(long addr, int len, int count) { + public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } @@ -208,7 +213,7 @@ public void appendRawEntries(long addr, int len, int count) { * (see the class-level durability note). Assigns the next dense id implicitly * (the entry's position). */ - public void appendSymbol(CharSequence symbol) { + public synchronized void appendSymbol(CharSequence symbol) { if (closed) { return; } @@ -244,7 +249,7 @@ public void appendSymbol(CharSequence symbol) { * range and overwrites at the same offset. No-op when the range is empty or * the dictionary is closed. */ - public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { + public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { if (closed || to < from) { return; } @@ -270,7 +275,7 @@ public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { } @Override - public void close() { + public synchronized void close() { if (closed) { return; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 222eeebe..58f9cfa9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -480,6 +480,70 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception }); } + @Test + public void testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating() throws Exception { + // Regression for the re-encode (else) branch of persistNewSymbolsBeforePublish. + // After a failed publish leaves the durable dictionary size ahead of the wire + // baseline (pd.size() > sentMaxSymbolId+1), a later flush that introduces a NEW + // symbol cannot reuse the frame's already-encoded fast-path bytes -- it + // re-encodes just the [pd.size() .. currentBatchMaxSymbolId] suffix via + // appendSymbols. Keying that off pd.size() (not sentMaxSymbolId+1) keeps it + // idempotent: the already-persisted prefix is NOT re-appended. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // Small segment: any frame carrying the padded row fails to publish with + // PAYLOAD_TOO_LARGE deterministically (no backpressure timing). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;"; + String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment + Sender sender = Sender.fromConfig(cfg); + try { + // Flush 1: s0 is persisted (write-ahead) before the oversized frame + // fails to publish. pd.size()=1, sentMaxSymbolId stays -1. + sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow(); + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + } + + // Add a NEW symbol s1 (id 1). The failed s0 row is still buffered, so + // the batch is {s0, s1} and the durable size (1) has run ahead of the + // wire baseline (-1) -- the state that selects the appendSymbols branch. + sender.table("m").symbol("s", "s1").stringColumn("p", pad).longColumn("v", 2L).atNow(); + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + } + + // The else branch persisted ONLY s1 (the suffix). The dictionary holds + // s0, s1 exactly once each. Pre-fix (appendSymbols from + // sentMaxSymbolId+1) re-appended s0, giving size 3. + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix", + 2, pd.size()); + Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); + Assert.assertEquals("s1", pd.readLoadedSymbols().getQuick(1)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized rows and fails + // again (PAYLOAD_TOO_LARGE); expected, resources still released. + } + } + } + }); + } + @Test public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index ab200e80..d413d82e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -656,6 +656,46 @@ public void testPoisonDwellHoldsEscalationUntilWallClockWindowElapses() throws E }); } + @Test + public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Exception { + // Regression: after a reconnect the loop ships the head DATA frame + // (dataFrameSentThisConnection=true) BEFORE tryReceiveAcks reads the server's + // NACK of a dictionary CATCH-UP frame in the same loop iteration. That NACK + // names a wire seq below the replay head, so its fsn = fsnAtZero+cappedSeq is + // at or below ackedFsn -- negative when replayStart < catchUpFrames. It must + // NOT charge a poison strike: recordHeadRejectionStrike(fsn) would set + // poisonFsn to that value (the common shape yields -1, the "no suspect" + // sentinel), laundering any genuine in-progress poison run and reporting a + // bogus fsn. The fix routes an fsn <= ackedFsn rejection to the pre-send path + // (surface + recycle, no strike), symmetric with the success path's + // engine.acknowledge() no-op below ackedFsn. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + // Model: 2 catch-up frames (wire seq 0,1) + 1 data frame (wire seq 2) + // sent, nothing acked (ackedFsn = -1), so fsnAtZero = replayStart(0) - + // catchUpFrames(2) = -2. A NACK of catch-up wire seq 0 maps to fsn -2, + // strictly below the -1 sentinel so the assertion discriminates. + setPostCatchUpDataFrameBaseline(loop, -2L, 3L); + assertEquals("precondition: no poison suspect yet", + -1L, getLongField(loop, "poisonFsn")); + + deliverRetriableNack(loop, 0, "disk full"); + + // The catch-up NACK was surfaced + recycled but charged NO strike, so + // the sentinel is intact. Pre-fix it was set to -2, laundering the + // poison detector's state. + assertEquals("catch-up NACK must not set/launder the poison sentinel", + -1L, getLongField(loop, "poisonFsn")); + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testPostSendNotWritableNackNeverEscalatesToPoisonTerminal() throws Exception { // RETRIABLE_OTHER (NOT_WRITABLE) is a node-state verdict, not a frame @@ -1058,6 +1098,28 @@ private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long cou f.setLong(loop, count); } + // Models a fresh connection that has shipped the dictionary catch-up (advancing + // fsnAtZero below the replay head) AND the head data frame: sets fsnAtZero, + // nextWireSeq, and dataFrameSentThisConnection=true together. + private static void setPostCatchUpDataFrameBaseline(CursorWebSocketSendLoop loop, + long fsnAtZero, long nextWireSeq) throws Exception { + Field z = CursorWebSocketSendLoop.class.getDeclaredField("fsnAtZero"); + z.setAccessible(true); + z.setLong(loop, fsnAtZero); + Field w = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); + w.setAccessible(true); + w.setLong(loop, nextWireSeq); + Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); + d.setAccessible(true); + d.setBoolean(loop, true); + } + + private static long getLongField(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getLong(loop); + } + private static long[] txns(long... v) { return v; } From fdd7141f246839f9df002d2a8e5b0c919bce02ba Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 14:37:42 +0100 Subject: [PATCH 29/78] Tidy delta symbol-dict dead code and edges Minor follow-ups from the review. Remove the unused CursorSendEngine.isMemoryMode() -- it had no callers; isDeltaDictEnabled() tests sfDir == null directly. Sort the NativeBufferWriter import into its alphabetical place among the qwp.client imports in CursorWebSocketSendLoop. Harden PersistedSymbolDict.openExisting: hoist entriesAddr / entriesLen so the catch frees the loaded-entries buffer too. It is unreachable today (nothing between that malloc and the return throws, and on success the buffer is transferred to the returned dict), but the error path no longer leaks if a future edit adds a throwing step. Return a defensive copy from DeltaDictCatchUpTest's dictFor() instead of the live inner list: the caller iterates it unlocked while the server thread may still be appending, matching the sibling dictSnapshot(). Add testTornDictionaryOneIdGapFailsCleanly -- the tightest torn-dictionary boundary (deltaStart == recoveredSize + 1), the common one-entry-short host-crash tail. The existing torn test uses a 3-id gap, so it does not pin the guard's false-negative edge; a "deltaStart > size + 1" mutation passes it but ships the gapped frame here (the new test fails on it). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 11 --- .../sf/cursor/CursorWebSocketSendLoop.java | 2 +- .../client/sf/cursor/PersistedSymbolDict.java | 12 ++- .../qwp/client/DeltaDictCatchUpTest.java | 8 +- .../qwp/client/DeltaDictRecoveryTest.java | 75 +++++++++++++++++++ 5 files changed, 92 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 4b692288..c962a56d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -629,17 +629,6 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } - /** - * True when this engine has no store-and-forward directory: the ring lives - * only in malloc'd memory, so it cannot be recovered after a crash and no - * orphan drainer ever replays it. Only in-process reconnect/failover replays - * its frames, which is what makes send-time symbol-dict catch-up (rather than - * fully self-sufficient frames) safe. - */ - public boolean isMemoryMode() { - return sfDir == null; - } - /** * Whether the sender may delta-encode symbol dictionaries on this engine. * Always true in memory mode (the send loop keeps an in-process catch-up diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 478d33c5..acb56cb7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -31,12 +31,12 @@ import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketUpgradeException; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; -import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index d2651923..889104ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -353,6 +353,8 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { return null; } long buf = 0L; + long entriesAddr = 0L; + int entriesLen = 0; try { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); @@ -388,8 +390,7 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { pos = next + (int) entryLen; count++; } - int entriesLen = pos - HEADER_SIZE; - long entriesAddr = 0L; + entriesLen = pos - HEADER_SIZE; if (entriesLen > 0) { entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); @@ -417,6 +418,13 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); } + // entriesAddr is transferred to the returned dict on the success path, + // and the catch only runs before that return, so freeing it here cannot + // double-free. Unreachable today (nothing between its malloc and the + // return throws), but keeps the error path leak-free against a future edit. + if (entriesAddr != 0L) { + Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + } Files.close(fd); LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); return null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 0d2ff9f7..d35a648e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -272,7 +272,9 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() - ? dictsByConn.get(connNumber - 1) + // Copy under the lock: the caller iterates it unlocked while the + // server thread may still be appending to the live inner list. + ? new ArrayList<>(dictsByConn.get(connNumber - 1)) : new ArrayList<>(); } @@ -406,7 +408,9 @@ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocke synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() - ? dictsByConn.get(connNumber - 1) + // Copy under the lock: the caller iterates it unlocked while the + // server thread may still be appending to the live inner list. + ? new ArrayList<>(dictsByConn.get(connNumber - 1)) : new ArrayList<>(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 58f9cfa9..1439ac20 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,81 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { + // Boundary variant of testTornDictionaryFailsCleanlyInsteadOfCorrupting: the + // recovered dictionary is short by EXACTLY ONE id -- the tightest gap the + // guard must still reject (deltaStart == recoveredSize + 1). A one-entry-short + // tail is the common host-crash outcome, so this pins the guard's + // false-negative edge: a "deltaStart > size + 1" mutation would let this + // single-id gap through and null-pad the missing symbol on the server. + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Lose the whole dictionary (truncate to the 8-byte header, size 0) but + // stamp the watermark at FSN 0, so recovery replays from FSN 1 -- a frame + // with deltaStart=1. The dictionary covers no ids, so id 0 is the single + // missing entry: deltaStart(1) == recoveredSize(0) + 1. Because size is 0 + // no catch-up is sent, so any counted frame would be the gapped data frame. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 0); + + // Phase 2: recover against a fresh counting server. The guard must fire on + // the very first replay frame (deltaStart 1 > recovered size 0) and fail + // terminally rather than send a frame that null-pads id 0. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertEquals("a one-id gap must still block replay to a fresh server", + 0, handler.frames.get()); + Assert.assertNotNull("a one-id-short dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("delta start 1 exceeds recovered dictionary size 0")); + } + }); + } + @Test public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's From 2fdbbf969e10bf7cef3986633bba599d8510e504 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 17:46:16 +0100 Subject: [PATCH 30/78] Fix catch-up terminal on a homogeneous batch cap The reconnect symbol-dictionary catch-up latched a terminal for any entry exceeding the packing budget (cap - HEADER_SIZE - 16). That reserve is larger than the minimal data-frame overhead, so a symbol the producer had already shipped under a given cap could exceed the budget and trip the terminal on reconnect -- permanently hard-failing a running producer on a homogeneous single-cap cluster, the exact failure the store-and-forward path exists to survive. sendDictCatchUp now tests the actual solo catch-up frame (header + the entry's delta varints + the entry bytes) against the cap. That frame is always smaller than the data frame that already carried the entry, so an accepted symbol always re-registers on the same cap; a genuinely oversized entry on a shrunk (heterogeneous) cap still terminates. The conservative packing budget stays, used only for multi-entry chunk splitting. Add a fixed-cap regression test: a 173-char symbol under a fixed cap of 200 is accepted, then re-registers gap-free across a reconnect. It fails before this change, where the reverted condition terminates a frame that fits the cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 38 +++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 61 +++++++++++++++++++ 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index acb56cb7..bd9a54bc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2171,12 +2171,21 @@ private long readVarintAt(long p, long limit) { */ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); - // Symbol-bytes budget per frame, leaving room for the 12-byte header and - // the two delta-section varints. cap <= 0 means the server advertised no - // limit -- send the whole dictionary in one frame, but still bound the - // budget by MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen - // (HEADER_SIZE + varints + symbolsLen) cannot overflow on a pathological - // multi-GB dictionary (unreachable at real cardinality; defensive). + // The frame ceiling a catch-up chunk must not exceed: the server's + // advertised cap, or -- when the server advertises none (cap <= 0) -- + // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE + + // varints + symbolsLen) cannot overflow on a pathological multi-GB + // dictionary (unreachable at real cardinality; defensive). Used by the + // single-entry terminal below, which measures the real solo frame. + int frameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES; + // Symbol-bytes budget for PACKING several entries into one chunk, leaving + // room for the 12-byte header and the two delta-section varints. Kept + // deliberately conservative (reserving 16 for the varints): it only makes a + // multi-entry chunk split marginally earlier, never over the cap. It must + // NOT gate the single-entry terminal -- that reserve is larger than the + // minimal data-frame overhead, so an entry the producer already shipped + // under this cap could exceed the reserve yet still fit its own catch-up + // frame; the terminal tests the real solo frame against frameLimit instead. int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; @@ -2192,7 +2201,20 @@ private int sendDictCatchUp() { long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] long entryBytes = entryEnd - entryStart; - if (entryBytes > budget) { + // The exact table-less frame sendCatchUpChunk would build for THIS entry + // alone: header + deltaStart varint (the entry's own global id) + + // deltaCount varint (1) + the entry bytes. Terminal only when even that + // solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // re-registered. Testing the real solo frame (not the conservative + // packing budget above) is what keeps a HOMOGENEOUS cluster + // livelock-free: an entry the producer already shipped in a data frame + // under this cap (header + delta varints + entry + schema + >=1 row) is + // strictly larger than its bare catch-up frame, so it always fits here. + long soloFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols) + + NativeBufferWriter.varintSize(1) + + entryBytes; + if (soloFrameLen > frameLimit) { // Non-retriable: the entry will not shrink and the same cluster // re-advertises the same cap, so reconnecting would livelock. // Latch a terminal (the data must be resent after the cap is @@ -2209,7 +2231,7 @@ private int sendDictCatchUp() { // the terminal keeps the homogeneous common case livelock-free. LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']'); recordFatal(err); throw new CatchUpSendException(err); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index d35a648e..811c0c67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -99,6 +99,67 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { }); } + @Test + public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exception { + // Regression (homogeneous single cap): a symbol whose length sits just below + // the advertised cap is ACCEPTED into a data frame (messageSize <= cap) and + // enters the sent-dictionary mirror. On reconnect the catch-up must + // re-register it under the SAME cap -- the bare catch-up frame (header + two + // varints + the entry) is smaller than the data frame that already shipped + // it (which also carried the table schema + a row), so it fits. + // + // Pre-fix, the single-entry terminal used the conservative PACKING budget + // (cap - HEADER_SIZE - 16), which is stricter than the producer's publish + // gate (messageSize <= cap) by more than the minimal data-frame overhead. So + // a symbol accepted onto the wire under cap C could exceed that budget and + // trip a spurious "during catch-up" terminal, permanently hard-failing a + // running producer on its first transient reconnect. Concretely at cap=200: + // table("t").symbol("s", <173 chars>).atNow() encodes to 198 bytes (<=200, + // accepted), its dict entry is 2+173=175 bytes (> old budget 172 -> old + // terminal), while the real solo catch-up frame is 12+1+1+175=189 (<=200 -> + // fits). Unlike testCatchUpEntryTooLargeForCapFailsTerminally (a genuinely + // oversized entry on a shrunk cap, which MUST still terminate), this entry + // is legally shippable and must NOT terminate. + final int cap = 200; + final String nearCapSymbol = TestUtils.repeat("x", 173); + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(cap); // same cap on every handshake (homogeneous) + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Symbol-only row so the near-cap symbol drives the frame size. + sender.table("t").symbol("s", nearCapSymbol).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + waitFor(() -> handler.conn1Closed, 5_000); + + // The reconnect runs the catch-up under the SAME cap. Pre-fix + // this latched a terminal (surfacing on this flush); post-fix the + // catch-up ships the near-cap symbol and the flush goes through. + sender.table("t").symbol("s", "beta").atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // Connection 2's dictionary, rebuilt purely from the frames it + // received, must hold the near-cap symbol (re-registered by the + // catch-up) and beta, gap-free -- proving the catch-up SHIPPED rather + // than terminated. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals(nearCapSymbol, conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + }); + } + @Test public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // A dictionary entry that exceeds the reconnect server's per-chunk budget From 8cfd7bdc86796c6a5c976d8da3d780a19e3d957d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 17:54:43 +0100 Subject: [PATCH 31/78] Fix recovery id desync on UTF-8-colliding symbols Two DISTINCT source symbols that collapse to the same UTF-8 bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get distinct producer ids and persist as separate .symbol-dict entries. On recovery, seedGlobalDictionaryFromPersisted replayed them through getOrAddSymbol, which de-dups the decoded "?" strings, leaving the producer dictionary (and sentMaxSymbolId) one short of pd.size(). That desynced the delta baseline from the send-loop catch-up mirror (which uses pd.size()) and, after a reconnect, silently misattributed later well-formed symbols. Add GlobalSymbolDictionary.addRecoveredSymbol, which appends at the next id WITHOUT de-duplicating, and seed recovery through it so the producer id space always matches the persisted entry count. The reverse lookup keeps the highest id for a colliding string, which is harmless -- both ids encode to identical bytes. A GlobalSymbolDictionary unit test pins the no-dedup contract, and a new recovery test seeds a slot with two lone-surrogate symbols and asserts the producer dictionary and sentMaxSymbolId match the persisted entry count. It fails before this change (dictionary size 1 vs the file's 2). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/GlobalSymbolDictionary.java | 31 ++++++++ .../qwp/client/QwpWebSocketSender.java | 12 +++- .../qwp/client/DeltaDictRecoveryTest.java | 71 +++++++++++++++++++ .../client/GlobalSymbolDictionaryTest.java | 30 ++++++++ 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 3b4c9a90..d1b0f13e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) { this.idToSymbol = new ObjList<>(initialCapacity); } + /** + * Appends {@code symbol} at the next sequential id, matching a recovered / + * persisted dictionary's dense id order, WITHOUT de-duplicating. + *

+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted}) + * replays the persisted entries in id order to rebuild this dictionary. It must + * NOT collapse two source strings that decode to the same characters, because + * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread + * catch-up mirror all key on the entry POSITION (id), not on the string. The + * only strings that collide this way are malformed lone UTF-16 surrogates, + * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would + * de-dup them and leave this dictionary SHORTER than the persisted entry count, + * desyncing the producer's delta baseline from the catch-up mirror (which uses + * {@code pd.size()}) and silently misattributing later symbols. Appending + * unconditionally keeps {@link #size()} equal to that count. The reverse lookup + * keeps the highest id for a colliding string, which is harmless: both ids + * encode to the same bytes, so resolving either is equivalent. + * + * @param symbol the recovered symbol string (must not be null) + * @return the id assigned (the previous {@link #size()}) + */ + public int addRecoveredSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + /** * Clears all symbols from the dictionary. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bb1256dd..23d1240f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3750,6 +3750,16 @@ private void resetSymbolDictStateForNewConnection() { * the slot's persisted dictionary (ids assigned in the same ascending order, * so they match the recovered frames) and resumes the delta baseline at the * recovered tip, so newly ingested symbols continue above the recovered ids. + *

+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): + * the persisted dictionary, the on-wire delta and the send-loop catch-up mirror + * all key on the entry POSITION (id), so the producer id space must match the + * persisted entry count exactly. {@code getOrAddSymbol} would collapse two + * source strings that decode to the same characters -- only malformed lone + * UTF-16 surrogates, which UTF-8-encode to {@code '?'} -- leaving this + * dictionary shorter than {@code pd.size()} and desyncing + * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()}, + * which silently misattributes later symbols after a reconnect. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { if (pd == null || pd.size() == 0) { @@ -3757,7 +3767,7 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { } ObjList symbols = pd.readLoadedSymbols(); for (int i = 0, n = symbols.size(); i < n; i++) { - globalSymbolDictionary.getOrAddSymbol(symbols.getQuick(i)); + globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); } sentMaxSymbolId = globalSymbolDictionary.size() - 1; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 1439ac20..e46d0148 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -36,6 +36,7 @@ import org.junit.Test; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; @@ -713,6 +714,76 @@ public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { }); } + @Test + public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Exception { + // M2 regression: two DISTINCT source symbols that collapse to the SAME UTF-8 + // bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get + // distinct producer ids and persist as separate entries. On recovery the + // producer must rebuild its id space to match the persisted entry count + // exactly. Pre-fix, seedGlobalDictionaryFromPersisted used getOrAddSymbol, + // which de-duped the two decoded "?" strings, leaving the producer + // dictionary (and sentMaxSymbolId) one short of pd.size() -- desyncing from + // the send-loop catch-up mirror (which uses pd.size()) and silently + // misattributing later symbols after a reconnect. addRecoveredSymbol appends + // without de-duping, keeping producer and mirror in lockstep. + assertMemoryLeak(() -> { + // Phase 1: ingest two lone-surrogate symbols in file mode, close-fast. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + s1.table("m").symbol("s", "\uD800").longColumn("v", 1L).atNow(); // lone surrogate -> '?' + s1.flush(); + s1.table("m").symbol("s", "\uD801").longColumn("v", 2L).atNow(); // a DIFFERENT one -> '?' + s1.flush(); + } + } + + // The persisted dictionary holds TWO entries (both encode to '?'). + int persistedSize; + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + persistedSize = pd.size(); + } finally { + pd.close(); + } + Assert.assertEquals("two colliding symbols must persist as two entries", 2, persistedSize); + + // Phase 2: recover. The seeded producer id space must match pd.size(). + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + Assert.assertEquals("recovered producer dictionary must match the persisted " + + "entry count (not de-duped), else the delta baseline desyncs from the mirror", + persistedSize, globalDictSize(s2)); + Assert.assertEquals("delta baseline must resume at the persisted tip", + persistedSize - 1, intField(s2, "sentMaxSymbolId")); + } + } + }); + } + + private static int globalDictSize(Sender sender) throws Exception { + Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); + f.setAccessible(true); + Object dict = f.get(sender); + return (int) dict.getClass().getMethod("size").invoke(dict); + } + + private static int intField(Sender sender, String name) throws Exception { + Field f = sender.getClass().getDeclaredField(name); + f.setAccessible(true); + return f.getInt(sender); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java index b8945d40..2f38ab02 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -31,6 +31,36 @@ public class GlobalSymbolDictionaryTest { + @Test + public void testAddRecoveredSymbol_appendsWithoutDeduplicating() { + // Recovery replays persisted entries in id order. Distinct source strings + // that decode to the same characters -- lone UTF-16 surrogates both + // UTF-8-encode to '?', so they read back as the string "?" -- must keep + // DISTINCT ids, so the producer id space matches the persisted entry count. + // getOrAddSymbol de-dups them; addRecoveredSymbol must not. + GlobalSymbolDictionary dedup = new GlobalSymbolDictionary(); + dedup.getOrAddSymbol("?"); + dedup.getOrAddSymbol("?"); + assertEquals("getOrAddSymbol de-dups colliding strings", 1, dedup.size()); + + GlobalSymbolDictionary recovered = new GlobalSymbolDictionary(); + assertEquals(0, recovered.addRecoveredSymbol("?")); + assertEquals(1, recovered.addRecoveredSymbol("?")); + assertEquals(2, recovered.addRecoveredSymbol("nvda")); + assertEquals("addRecoveredSymbol keeps colliding entries distinct", 3, recovered.size()); + + // Dense id -> symbol mapping is preserved position-for-position. + assertEquals("?", recovered.getSymbol(0)); + assertEquals("?", recovered.getSymbol(1)); + assertEquals("nvda", recovered.getSymbol(2)); + + // A later ingest of a colliding string reuses the highest recovered id + // (harmless -- both encode to identical bytes), and a genuinely new symbol + // continues past the recovered tip. + assertEquals(1, recovered.getOrAddSymbol("?")); + assertEquals(3, recovered.getOrAddSymbol("brand-new")); + } + @Test public void testAddSymbol_assignsSequentialIds() { GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); From 0b4ab3ff1c0ac5a044da8eb49c6a69fbb7528a72 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 18:06:40 +0100 Subject: [PATCH 32/78] Harden symbol-dict resource teardown paths Three defensive fixes on currently-unreachable teardown paths, each guarding a future edit from a native-memory hazard: - PersistedSymbolDict.close() now nulls loadedEntriesAddr/Len after freeing them (like scratchAddr), so a post-close read of the non-closed-guarded getters cannot dereference freed memory. - CursorWebSocketSendLoop resets sentDictCount alongside the buffer in both mirror-free sites, keeping the mirror all-or-nothing. start() has no closed guard, so a hypothetical close()-then-start() would otherwise observe a non-zero count against a freed buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. - PersistedSymbolDict.openFresh() now closes the fd on any throw (the header malloc moves inside the try, plus a catch), mirroring openExisting; previously a throw in the header-write body freed the scratch buffer but leaked the fd. Tests: testCloseNullsLoadedEntries asserts the getters read 0 after close; the mirror-leak test now also asserts sentDictCount is reset. Both fail before their fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 6 +++++ .../client/sf/cursor/PersistedSymbolDict.java | 20 ++++++++++++-- ...CursorWebSocketSendLoopMirrorLeakTest.java | 8 ++++++ .../sf/cursor/PersistedSymbolDictTest.java | 26 +++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index bd9a54bc..7593cf47 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -907,6 +907,11 @@ public synchronized void close() { sentDictBytesAddr = 0; sentDictBytesCapacity = 0; sentDictBytesLen = 0; + // Reset the count alongside the buffer so the mirror stays all-or- + // nothing: a hypothetical close()-then-start() (start() has no closed + // guard) must not observe a non-zero sentDictCount against a freed + // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. + sentDictCount = 0; } } @@ -1793,6 +1798,7 @@ private void ioLoop() { sentDictBytesAddr = 0; sentDictBytesCapacity = 0; sentDictBytesLen = 0; + sentDictCount = 0; // keep the mirror all-or-nothing (see close()) } shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 889104ec..7f557708 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -282,6 +282,11 @@ public synchronized void close() { closed = true; if (loadedEntriesAddr != 0L) { Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT); + // Null after freeing (like scratchAddr below) so a future accessor that + // reads loadedEntriesAddr()/loadedEntriesLen() post-close cannot + // dereference freed native memory; the getters are not closed-guarded. + loadedEntriesAddr = 0L; + loadedEntriesLen = 0; } if (scratchAddr != 0L) { Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); @@ -437,8 +442,9 @@ private static PersistedSymbolDict openFresh(String filePath) { LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); return null; } - long hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + long hdr = 0L; try { + hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC); Unsafe.getUnsafe().putByte(hdr + 4, VERSION); Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); @@ -451,8 +457,18 @@ private static PersistedSymbolDict openFresh(String filePath) { LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; } + } catch (Throwable t) { + // Unreachable today (Files.write is native and returns -1 rather than + // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte + // malloc cannot realistically OOM), but close the fd against a future + // edit so it cannot leak -- mirroring openExisting's error handling. + Files.close(fd); + LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); + return null; } finally { - Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + if (hdr != 0L) { + Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + } } return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 7869882d..057e1a5f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -89,7 +89,15 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { 0, 0, 1); // Close without start(): the ctor-seeded mirror is this // thread's to free, since the I/O loop never ran. + Assert.assertTrue("precondition: the ctor seeded a non-empty mirror", + readInt(loop, "sentDictCount") > 0); loop.close(); + // close() must reset sentDictCount alongside freeing the buffer, + // so the mirror stays all-or-nothing: a hypothetical post-close + // start() (no closed guard) cannot read a stale count against a + // freed buffer and drive a null-mirror catch-up. + Assert.assertEquals("close() must reset sentDictCount to 0", + 0, readInt(loop, "sentDictCount")); } }); } finally { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index cc45c23b..aa37031b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -228,6 +228,32 @@ public void testBadMagicIsRecreatedEmpty() throws Exception { }); } + @Test + public void testCloseNullsLoadedEntries() throws Exception { + // close() must null loadedEntriesAddr/Len after freeing them (like + // scratchAddr), so an accidental post-close read of the getters cannot + // dereference freed native memory. Pre-fix the pointer survived close() + // non-zero. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("AAPL"); + d.close(); + + // Reopen so recovery loads the entries into native memory. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertTrue("recovery must load entries into native memory", + re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0); + re.close(); + Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr()); + Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen()); + } finally { + rmDir(dir); + } + }); + } + @Test public void testEmptySymbolRoundTrips() throws Exception { assertMemoryLeak(() -> { From 186ae10dec23ff02e982b87a49047b33764d1a24 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 18:15:51 +0100 Subject: [PATCH 33/78] Defer catch-up commit; fix stale delta comments Three delta symbol-dict cleanups (a fourth reported item, a dead CursorSendEngine.isMemoryMode(), was already removed earlier on this branch, so nothing to do there): - sendCatchUpChunk now sets FLAG_DEFER_COMMIT on the catch-up frame. It carries dictionary entries but no rows, so it must never trigger a server-side commit. Today it is always the first frame on a fresh (empty-transaction) connection, so committing nothing is a no-op -- but that invariant is load-bearing and was unasserted. Deferring the empty commit removes the dependency, so a future mid-stream catch-up cannot prematurely commit an in-flight deferred transaction. The dictionary delta still registers, as any deferred data frame's does. - flushPendingRows' comment claimed delta mode is "memory-mode only"; file-mode store-and-forward ships monotonic deltas too once the persisted dictionary opened. Corrected. - The deltaDictEnabled field comment implied delta mode only applies to recovered / orphan-drained disk slots; it applies to fresh disk slots too. Clarified that recovery additionally seeds the mirror. testReconnectCatchUpRebuildsDictionary now asserts the catch-up frame carries FLAG_DEFER_COMMIT; it fails without the flag. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 3 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 19 ++++++++++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 8 ++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 23d1240f..5915f0fb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3455,7 +3455,8 @@ private void flushPendingRows(boolean deferCommit) { ensureActiveBufferReady(); // In full-dict mode every frame is self-sufficient: it carries the whole // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh - // server never dangles a symbol id. In delta mode (memory-mode only) each + // server never dangles a symbol id. In delta mode (memory-mode, and + // file-mode store-and-forward once the persisted dictionary opened) each // frame carries only ids above sentMaxSymbolId; a reconnect re-registers // the dictionary via an I/O-thread catch-up frame before replay, so the // producer's monotonic baseline stays valid across the wire boundary. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 7593cf47..87d1dab0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -187,8 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); // Delta symbol dictionary catch-up state (see swapClient). Active in memory - // mode, and in disk mode on a recovered / orphan-drained slot (the constructor - // seeds sentDict* from the persisted dictionary there). + // mode, and in disk mode whenever the per-slot persisted dictionary opened -- + // fresh slots included, not just recovered / orphan-drained ones. On a + // recovered / orphan-drained slot the constructor additionally SEEDS sentDict* + // from that persisted dictionary; a fresh slot starts with an empty mirror and + // grows it as frames are sent. // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that @@ -2293,8 +2296,18 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO + // rows, so it must never trigger a server-side commit. Today it is + // always the first frame on a fresh (empty-transaction) connection, so + // committing nothing is a no-op -- but that invariant is load-bearing + // and unasserted. Deferring the (empty) commit removes the dependency: + // a future mid-stream catch-up cannot prematurely commit an in-flight + // deferred transaction. The dictionary delta still registers (as any + // deferred data frame's does); only the row commit is deferred, and the + // next real frame commits it. Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, - (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT)); + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT + | QwpConstants.FLAG_DEFER_COMMIT)); Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount Unsafe.getUnsafe().putInt(frame + 8, payloadLen); long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 811c0c67..e7f27db1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -92,6 +92,9 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { List conn2 = handler.dictFor(2); Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", handler.sawZeroTableFrameOnConn2); + Assert.assertTrue("the catch-up frame carries no rows, so it must defer its " + + "(empty) commit -- FLAG_DEFER_COMMIT set", + handler.catchUpDeferredOnConn2); Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); Assert.assertEquals("alpha", conn2.get(0)); Assert.assertEquals("beta", conn2.get(1)); @@ -326,6 +329,9 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot // race into connection 1's pre-close window and must land on the reconnect. volatile boolean conn1Closed; + // Set from the flags byte of the zero-table catch-up frame on connection 2: + // the catch-up carries no rows and must defer its (empty) commit. + volatile boolean catchUpDeferredOnConn2; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -352,6 +358,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien accumulate(data, dict); if (connNumber == 2 && tableCount(data) == 0) { sawZeroTableFrameOnConn2 = true; + // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). + catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; } try { client.sendBinary(buildAck(nextSeq.getAndIncrement())); From fef7203593c13a1c8308b91afe081d91ee0779dd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 19:34:11 +0100 Subject: [PATCH 34/78] Plug ensureConnected leak; consolidate varint helpers Review cleanups: - ensureConnected's start()-failure catch now closes cursorSendLoop before nulling client. Previously it closed only the client; because the connected flag is set past the catch, a caller that retries -- re-entering ensureConnected and reassigning cursorSendLoop -- orphaned a recovered slot's ctor-seeded native mirror. That is a reachable leak on the recovered-slot + start()/dispatcher-OOM + retry path. close() is idempotent and frees the mirror via its loopNeverRan path, and also closes the shared client, so the following client.close() is a no-op. - Consolidated the triplicated raw-address varint writer into NativeBufferWriter.writeVarint (writeVarintDirect now delegates to it); PersistedSymbolDict and the catch-up frame builder in CursorWebSocketSendLoop use it, and PersistedSymbolDict.varintSize now reuses NativeBufferWriter.varintSize. - Alphabetized CursorSendEngine (getPersistedSymbolDict before getTotalBackpressureStalls before isDeltaDictEnabled) and moved PersistedSymbolDict.decodeVarint to the front of its private-static cluster. - readVarintAt gained the shift > 35 defensive bound that decodeVarint already has (unreachable today, all inputs are freshly-encoded or validated varints, but consistent). - CursorWebSocketSendLoopCatchUpAlignmentTest now uses the existing TestUtils.createTmpDir/removeTmpDir instead of reinventing them. Not changed: the reported dead CursorSendEngine.isMemoryMode() and a PersistedSymbolDict MAX_ENTRY_LEN constant do not exist at HEAD (the former removed earlier on this branch, the latter never present), and the NativeBufferWriter import is already correctly ordered. The buildAck/waitFor/rmDir/readVarint test helpers are duplicated across ~19 pre-existing test files, so consolidating them belongs in a dedicated repo-wide test-hygiene change rather than this feature PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/NativeBufferWriter.java | 21 ++++-- .../qwp/client/QwpWebSocketSender.java | 11 +++ .../client/sf/cursor/CursorSendEngine.java | 18 ++--- .../sf/cursor/CursorWebSocketSendLoop.java | 21 +++--- .../client/sf/cursor/PersistedSymbolDict.java | 75 +++++++------------ ...WebSocketSendLoopCatchUpAlignmentTest.java | 24 +----- 6 files changed, 77 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index aad95f3c..cb6ebb92 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -76,6 +76,21 @@ public static int varintSize(long value) { return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; } + /** + * Writes {@code value} as an unsigned LEB128 varint directly at native address + * {@code addr} and returns the address just past the last byte. The canonical + * raw-address varint writer shared by the SF cursor's persisted dictionary and + * catch-up frame builder. + */ + public static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + @Override public void close() { if (bufferPtr != 0) { @@ -336,11 +351,7 @@ public void skip(int bytes) { } private static void writeVarintDirect(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr, (byte) value); + writeVarint(addr, value); } private void encodeUtf8(CharSequence value, int utf8Len) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 5915f0fb..d00ab872 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3362,6 +3362,17 @@ private void ensureConnected() { cursorSendLoop.setConnectionDispatcher(connectionDispatcher); cursorSendLoop.start(); } catch (Throwable t) { + // start() (or dispatcher construction) failed after cursorSendLoop was + // assigned. Close it so a caller that retries -- re-entering + // ensureConnected and reassigning cursorSendLoop above -- cannot orphan + // a recovered slot's ctor-seeded native mirror (freed only by close() + // or the I/O loop, neither of which has run). close() is idempotent and + // frees the mirror via its loopNeverRan path; it also closes the shared + // client, so the client.close() below is a safe idempotent no-op. + if (cursorSendLoop != null) { + cursorSendLoop.close(); + cursorSendLoop = null; + } if (client != null) { client.close(); client = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index c962a56d..6c34124d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -619,6 +619,15 @@ public MmapSegment firstSealed() { return ring.firstSealed(); } + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Number of times {@link #appendBlocking} hit * {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and @@ -641,15 +650,6 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } - /** - * The engine's persisted symbol dictionary, or {@code null} in memory mode - * (and in disk mode if it failed to open). The producer appends new symbols - * to it; recovery / orphan-drain read its loaded entries to seed catch-up. - */ - public PersistedSymbolDict getPersistedSymbolDict() { - return persistedSymbolDict; - } - /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 87d1dab0..a855cc50 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2161,6 +2161,14 @@ private long readVarintAt(long p, long limit) { break; } shift += 7; + if (shift > 35) { + // Defensive bound, matching PersistedSymbolDict.decodeVarint: a + // canonical entry-length / delta varint is <= 5 bytes. Every caller + // reads freshly-encoded, CRC- or openExisting-validated bytes, so + // this is unreachable, but it stops a corrupt continuation run from + // over-shifting into a garbage length. + break; + } } varintEnd = cur; return value; @@ -2310,8 +2318,8 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, | QwpConstants.FLAG_DEFER_COMMIT)); Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount Unsafe.getUnsafe().putInt(frame + 8, payloadLen); - long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); - q = writeVarintAt(q, deltaCount); + long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); + q = NativeBufferWriter.writeVarint(q, deltaCount); Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); client.sendBinary(frame, frameLen); } catch (Throwable t) { @@ -2331,15 +2339,6 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, totalFramesSent.incrementAndGet(); } - private static long writeVarintAt(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr++, (byte) value); - return addr; - } - private boolean tryReceiveAcks() { boolean any = false; try { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 7f557708..f00acbd4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -218,10 +219,10 @@ public synchronized void appendSymbol(CharSequence symbol) { return; } int utf8Len = Utf8s.utf8Bytes(symbol); - int varLen = varintSize(utf8Len); + int varLen = NativeBufferWriter.varintSize(utf8Len); int recLen = varLen + utf8Len; ensureScratch(recLen); - long p = writeVarint(scratchAddr, utf8Len); + long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } @@ -257,9 +258,9 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); - int recLen = varintSize(utf8Len) + utf8Len; + int recLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; ensureScratch(len + recLen); - long p = writeVarint(scratchAddr + len, utf8Len); + long p = NativeBufferWriter.writeVarint(scratchAddr + len, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } @@ -351,6 +352,30 @@ public int size() { return size; } + /** + * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns + * {@code [value, newPos]} or {@code null} if the varint is truncated + * (torn tail). + */ + private static long[] decodeVarint(long buf, int pos, int limit) { + long value = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return new long[]{value, cur}; + } + shift += 7; + if (shift > 35) { + return null; // implausible for an entry length; treat as torn + } + } + return null; + } + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { @@ -473,48 +498,6 @@ private static PersistedSymbolDict openFresh(String filePath) { return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); } - /** - * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns - * {@code [value, newPos]} or {@code null} if the varint is truncated - * (torn tail). - */ - private static long[] decodeVarint(long buf, int pos, int limit) { - long value = 0; - int shift = 0; - int cur = pos; - while (cur < limit) { - byte b = Unsafe.getUnsafe().getByte(buf + cur); - cur++; - value |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return new long[]{value, cur}; - } - shift += 7; - if (shift > 35) { - return null; // implausible for an entry length; treat as torn - } - } - return null; - } - - private static int varintSize(long value) { - int n = 1; - while (value > 0x7F) { - value >>>= 7; - n++; - } - return n; - } - - private static long writeVarint(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr++, (byte) value); - return addr; - } - private void ensureScratch(int required) { if (scratchCap >= required) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 79640f25..3814d826 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -30,7 +30,6 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.network.PlainSocketFactory; -import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; @@ -42,7 +41,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -78,30 +76,12 @@ public class CursorWebSocketSendLoopCatchUpAlignmentTest { @Before public void setUp() { - tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), - "qdb-cursor-catchup-" + System.nanoTime()).toString(); - assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + tmpDir = TestUtils.createTmpDir("qdb-cursor-catchup-"); } @After public void tearDown() { - if (tmpDir == null) return; - long find = Files.findFirst(tmpDir); - if (find > 0) { - try { - int rc = 1; - while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - if (name != null && !".".equals(name) && !"..".equals(name)) { - Files.remove(tmpDir + "/" + name); - } - rc = Files.findNext(find); - } - } finally { - Files.findClose(find); - } - } - Files.remove(tmpDir); + TestUtils.removeTmpDir(tmpDir); } @Test From cbaf474c270d16a45f8041cfdc03fe40855f11a6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 21:18:12 +0100 Subject: [PATCH 35/78] Add a per-entry CRC to the SF symbol dictionary The .symbol-dict side-file was page-cache durable but carried no integrity check, so a host-crash interior tear -- a lost page reading back as zeroes, or a stale entry left past the end by a failed truncate -- could shift the dense id->symbol map without changing the entry count the send-loop replay guard checks, silently misattributing symbol-column values on recovery. The SF segment frames are CRC-32C protected; the dictionary they depend on was not. Each entry now carries a trailing CRC-32C over its [len][utf8] bytes, the same checksum MmapSegment uses. openExisting verifies every entry and stops at the first torn or mismatched one, so recovery trusts only the intact prefix and the guard then forces a resend of the rest -- turning a silent misattribution into a fail-clean "resend required". open() strips the CRCs into the wire-shaped loaded-entries buffer, so the catch-up mirror seed and appendRawEntries keep the file==wire contract and the send loop stays untouched. The tail truncate is now failure-checked: a file open() cannot trim is untrusted and recreated empty rather than left exposing stale bytes. Bumps the on-disk format to v2 with a version check; existing files are recreated. A tear that leaves a CRC-matching byte run is still undetected, a 1-in-2^32 collision per entry, no weaker than the frames. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 246 ++++++++++++------ .../sf/cursor/PersistedSymbolDictTest.java | 56 ++++ 2 files changed, 222 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index f00acbd4..6bdc8162 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; +import io.questdb.client.std.Crc32c; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -53,12 +54,15 @@ * Layout (little-endian): * 

  *   offset 0: u32 magic = 'SYD1'
- *   offset 4: u8  version = 1
+ *   offset 4: u8  version = 2
  *   offset 5: 3 bytes reserved (zero)
- *   offset 8: entries, each [len: varint][utf8 bytes], in ascending global-id order
+ *   offset 8: entries, each [len: varint][utf8 bytes][crc32c: u32], in ascending global-id order
  *
* Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned - * sequentially from 0), so no id needs to be stored. + * sequentially from 0), so no id needs to be stored. Each entry carries a + * CRC-32C over its {@code [len][utf8]} bytes (the same checksum the SF segment + * frames use), so a torn or stale entry is detected on recovery instead of + * being silently mis-parsed. *

* Durability / write-ahead ordering: the producer appends the symbols a * frame introduces BEFORE that frame is published to the ring, but does NOT @@ -68,18 +72,27 @@ * dictionary is a superset of every recoverable frame's references. It is NOT * sufficient for a host/power crash, where unflushed pages can be lost out * of order and the dictionary may end up torn relative to the frames it serves -- - * exactly as the segment frames themselves may be lost on a host crash. The send - * loop's replay guard catches the COMMON host-crash outcome -- a truncated tail, - * where a frame's delta start id exceeds the recovered dictionary size -- and - * fails loudly (the unreplayable data must be resent) rather than sending a - * gapped frame. It does NOT catch every host-crash tear: an interior page lost - * out of order reads back as zeroes that parse as empty-string entries, and a - * failed best-effort truncate (see {@link #open}) can leave a stale trailing - * entry -- either SHIFTS the dense id->symbol mapping without changing the entry - * count the guard checks, so a replay silently misattributes symbols. That - * residual exposure sits within the "not host-crash durable" boundary above; a - * per-entry or running CRC would be needed to close it (the 3 reserved header - * bytes leave room for a future integrity field). + * exactly as the segment frames themselves may be lost on a host crash. Two + * layers keep a host-crash tear from silently corrupting data: + *

    + *
  • The per-entry CRC-32C: {@link #open} verifies every entry and stops at + * the first one whose checksum fails, so an interior page lost out of + * order (reading back as zeroes) or a stale entry left past the end by a + * failed truncate is DETECTED and the trusted region ends before it -- + * recovery never mis-parses a corrupt entry as a real symbol nor shifts + * the dense id->symbol map. The truncate that drops a torn/stale tail is + * now failure-checked (see {@link #open}): a file that cannot be trimmed + * is untrusted and recreated empty rather than left exposing stale bytes.
    • + *
  • The send loop's replay guard: once recovery trusts only the intact + * prefix, a surviving frame whose delta start id exceeds that prefix + * fails loudly (the unreplayable data must be resent) rather than sending + * a gapped frame.
    • + *
+ * Together these turn every detectable host-crash tear into a fail-clean + * "resend required" instead of a silent symbol misattribution -- the same + * CRC-32C protection the segment frames carry. A tear that happened to leave a + * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32 + * collision per corrupted entry, no weaker than the frames' own checksum. *

* A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. @@ -101,9 +114,10 @@ public final class PersistedSymbolDict implements QuietCloseable { * OrphanScanner, trim) skip it automatically. */ public static final String FILE_NAME = ".symbol-dict"; + static final int CRC_SIZE = 4; // u32 CRC-32C trailing every entry static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; - static final byte VERSION = 1; + static final byte VERSION = 2; // v2 appended the per-entry CRC-32C private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; private long appendOffset; @@ -184,26 +198,56 @@ public static void removeOrphan(String slotDir) { } /** - * Appends {@code count} already-encoded entries -- {@code [len varint][utf8]...}, - * the exact layout {@link #appendSymbols} produces -- verbatim from {@code addr} - * in a single write. The producer uses this when it already holds those bytes - * (the symbol-dict delta section the frame encoder just wrote), so it does not - * re-encode the same symbols. Writes {@code len} bytes and advances {@code size} - * by {@code count}. Same durability/idempotency contract as {@link #appendSymbols}: - * no fsync, and a short write throws WITHOUT advancing {@code size}/{@code - * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range - * at the same offset. No-op when the range is empty or the dictionary is closed. + * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the + * symbol-dict delta section the frame encoder just wrote -- to the on-disk + * dictionary, computing and appending a per-entry CRC-32C as it copies so the + * producer does not re-encode the symbols. The on-disk layout is + * {@code [len varint][utf8][crc32c]} per entry (see the class layout note); the + * {@code addr}/{@code len} bytes carry no CRC, so this walks the {@code count} + * entries to insert one. Advances {@code size} by {@code count}. Same + * durability/idempotency contract as {@link #appendSymbols}: no fsync, and a + * short write throws WITHOUT advancing {@code size}/{@code appendOffset}, so a + * retry keyed off {@link #size()} re-persists the same range at the same + * offset. No-op when the range is empty or the dictionary is closed. */ public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } - long written = Files.write(fd, addr, len, appendOffset); - if (written != len) { + int outLen = len + count * CRC_SIZE; + ensureScratch(outLen); + long src = addr; + long srcLimit = addr + len; + long dst = scratchAddr; + for (int i = 0; i < count; i++) { + long entryStart = src; + long symLen = 0; + int shift = 0; + while (src < srcLimit) { + byte b = Unsafe.getUnsafe().getByte(src++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + long entryEnd = src + symLen; // src is just past the len varint + if (entryEnd > srcLimit) { + throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME + + " [entry=" + i + ", count=" + count + ']'); + } + int wireSpan = (int) (entryEnd - entryStart); // [len][utf8] + Unsafe.getUnsafe().copyMemory(entryStart, dst, wireSpan); + Unsafe.getUnsafe().putInt(dst + wireSpan, Crc32c.update(Crc32c.INIT, entryStart, wireSpan)); + dst += wireSpan + CRC_SIZE; + src = entryEnd; + } + long written = Files.write(fd, scratchAddr, outLen, appendOffset); + if (written != outLen) { throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + len + ", actual=" + written + ']'); + + " [expected=" + outLen + ", actual=" + written + ']'); } - appendOffset += len; + appendOffset += outLen; size += count; } @@ -212,7 +256,9 @@ public synchronized void appendRawEntries(long addr, int len, int count) { * frame's new symbols BEFORE publishing that frame, so the write ordering * (dictionary entry before referencing frame) holds; no fsync is performed * (see the class-level durability note). Assigns the next dense id implicitly - * (the entry's position). + * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC + * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on + * recovery. */ public synchronized void appendSymbol(CharSequence symbol) { if (closed) { @@ -220,12 +266,14 @@ public synchronized void appendSymbol(CharSequence symbol) { } int utf8Len = Utf8s.utf8Bytes(symbol); int varLen = NativeBufferWriter.varintSize(utf8Len); - int recLen = varLen + utf8Len; + int wireLen = varLen + utf8Len; // [len][utf8] + int recLen = wireLen + CRC_SIZE; // + trailing crc ensureScratch(recLen); long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } + Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen)); long written = Files.write(fd, scratchAddr, recLen, appendOffset); if (written != recLen) { throw new IllegalStateException("short write to " + FILE_NAME @@ -237,11 +285,12 @@ public synchronized void appendSymbol(CharSequence symbol) { /** * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes - * the whole {@code [len varint][utf8]...} region into scratch first, then - * issues one positioned write -- versus one {@code pwrite(2)} per symbol via - * {@link #appendSymbol}. That per-symbol syscall count is the hot-path cost - * on a high-cardinality batch (one new symbol per row), which is exactly the - * store-and-forward workload delta encoding targets. Callers pass the + * the whole {@code [len varint][utf8][crc32c]...} region into scratch first, + * then issues one positioned write -- versus one {@code pwrite(2)} per symbol + * via {@link #appendSymbol}. That per-symbol syscall count is the hot-path + * cost on a high-cardinality batch (one new symbol per row), which is exactly + * the store-and-forward workload delta encoding targets. Each entry carries a + * trailing CRC-32C over its {@code [len][utf8]} bytes. Callers pass the * dictionary and the range so the ids resolve to their symbol strings. *

* Same durability and idempotency contract as {@link #appendSymbol}: no @@ -258,13 +307,15 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); - int recLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; - ensureScratch(len + recLen); - long p = NativeBufferWriter.writeVarint(scratchAddr + len, utf8Len); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + ensureScratch(len + wireLen + CRC_SIZE); + long entryStart = scratchAddr + len; + long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } - len += recLen; + Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen)); + len += wireLen + CRC_SIZE; } long written = Files.write(fd, scratchAddr, len, appendOffset); if (written != len) { @@ -389,61 +440,96 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); long read = Files.read(fd, buf, len, 0); - if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC) { - LOG.warn("symbol dict {} unreadable or bad magic; recreating", filePath); + if (read != len + || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC + || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { + LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); Files.close(fd); return null; } - // Parse complete entries starting after the header; stop at the - // first torn/incomplete trailing entry. - int pos = HEADER_SIZE; + // Parse complete, CRC-valid entries after the header; stop at the first + // torn/incomplete OR crc-mismatched entry. The CRC turns an interior + // tear (a lost page reading back as zeroes) or a stale entry left past + // the end by a failed truncate into a clean stop point, so recovery + // trusts only the intact prefix instead of silently mis-parsing a + // corrupt entry and shifting the dense id->symbol map. + int diskPos = HEADER_SIZE; // walks the on-disk [len][utf8][crc] entries int count = 0; - while (pos < len) { - long[] vr = decodeVarint(buf, pos, len); + int wireLen = 0; // running size of the crc-stripped copy + while (diskPos < len) { + long[] vr = decodeVarint(buf, diskPos, len); if (vr == null) { break; // torn length varint } - long entryLen = vr[0]; - int next = (int) vr[1]; - // The file length is the sole sanity bound: a length that runs past - // the file is a torn/incomplete trailing entry and stops the parse - // (self-healing tail). There is deliberately NO fixed per-entry - // ceiling -- the write path (appendSymbol/appendSymbols) applies none - // either, so a legitimately persisted large symbol must recover here, - // not be truncated and then trip the send loop's "resend required" - // guard on a normal process-crash recovery. entryLen stays a long so a - // corrupt multi-gigabyte length cannot wrap an int back under the check. - if ((long) next + entryLen > len) { - break; // torn/incomplete trailing entry -- self-healing tail + long symLen = vr[0]; + int afterVar = (int) vr[1]; + // [len varint][utf8] then a u32 CRC. symLen stays a long so a + // corrupt multi-gigabyte length cannot wrap an int back under the + // bound check. No fixed per-entry ceiling -- the write path applies + // none, so a legitimately large symbol must recover here. + long wireEnd = (long) afterVar + symLen; // end of [len][utf8] + if (wireEnd + CRC_SIZE > len) { + break; // torn/incomplete trailing entry (its CRC doesn't fit) + } + int wireEndI = (int) wireEnd; + int wireSpan = wireEndI - diskPos; + int crcStored = Unsafe.getUnsafe().getInt(buf + wireEndI); + int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, wireSpan); + if (crcCalc != crcStored) { + break; // corrupt/stale entry -- stop before it (fail-clean) } - pos = next + (int) entryLen; + diskPos = wireEndI + CRC_SIZE; + wireLen += wireSpan; count++; } - entriesLen = pos - HEADER_SIZE; - if (entriesLen > 0) { - entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); - Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); + int diskConsumed = diskPos - HEADER_SIZE; // valid entries incl. their CRCs + // Materialise the trusted entries as WIRE bytes ([len][utf8]..., no + // CRC) so loadedEntries*/readLoadedSymbols and the send-loop catch-up + // mirror stay wire-shaped -- the on-disk CRC is stripped here, once, at + // open. A second no-alloc walk over the already-validated region. + if (wireLen > 0) { + entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT); + long dst = entriesAddr; + int p = HEADER_SIZE; + for (int i = 0; i < count; i++) { + int vp = p; + long symLen = 0; + int shift = 0; + while (true) { + byte b = Unsafe.getUnsafe().getByte(buf + vp++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC + Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan); + dst += wireSpan; + p += wireSpan + CRC_SIZE; // skip the entry's CRC + } } + entriesLen = wireLen; Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; - // Physically drop any torn/stale trailing bytes (a crash mid-append) - // so a LATER, shorter append cannot leave residue past its own end - // that a future recovery mis-parses as a ghost symbol -- which would - // shift every subsequent dense id. Without this, appendOffset already - // lands just past the last complete entry, so the next append - // overwrites FROM there, but only truncation guarantees nothing - // survives BEYOND it. Best-effort: a failed truncate (rare -- an I/O - // error) leaves a latent silent-corruption path, not merely a lost - // optimization: if a later, shorter append then leaves stale bytes past - // its end, a subsequent recovery mis-parses them as a ghost symbol and - // shifts the dense ids, and the count-based replay guard does not catch - // it (see the class-level durability note). - long validLen = HEADER_SIZE + entriesLen; - if (validLen < len) { - Files.truncate(fd, validLen); + // Drop any torn/stale trailing bytes so a LATER, shorter append cannot + // leave residue past its own end. Unlike before, the truncate result IS + // checked: a file we cannot trim could still expose stale post-end bytes + // whose (self-consistent) per-entry CRC the parse would accept at a + // shifted position, so a failed truncate makes the file untrusted -- + // open() then recreates it empty (fail-clean) rather than risk a silent + // misattribution. + long validLen = HEADER_SIZE + diskConsumed; + if (validLen < len && !Files.truncate(fd, validLen)) { + LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); + if (entriesAddr != 0L) { + Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + } + Files.close(fd); + return null; } - return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); + return new PersistedSymbolDict(fd, validLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index aa37031b..cfcf4cf5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -281,6 +281,62 @@ public void testEmptySymbolRoundTrips() throws Exception { }); } + @Test + public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exception { + // A host-crash interior tear (a lost page reading back as zeroes) or a + // stale entry left past the end by a failed truncate can change the bytes + // of a NON-trailing entry. Without the per-entry CRC the parse would + // accept those bytes, shifting the dense id->symbol map and silently + // misattributing symbol-column values on replay. With the CRC the corrupt + // entry fails verification and the parse stops there, so recovery trusts + // only the intact prefix (fail-clean: the send loop's torn-dict guard then + // forces a resend of the rest). + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol("s0"); + d.appendSymbol("s1"); + d.appendSymbol("s2"); + d.appendSymbol("s3"); + d.appendSymbol("s4"); + Assert.assertEquals(5, d.size()); + } finally { + d.close(); + } + + // Corrupt one byte inside the 3rd entry's UTF-8 (id 2). On-disk + // entry layout is [len varint][utf8][crc32c u32]; a 2-byte ASCII + // symbol is 1 + 2 + 4 = 7 bytes, after the 8-byte header: + // header[0,8) e0[8,15) e1[15,22) e2[22,29) ... + // Offset 23 is "s2"'s first UTF-8 byte; flipping it leaves e2's + // stored CRC stale. + Path f = dir.resolve(".symbol-dict"); + byte[] bytes = Files.readAllBytes(f); + bytes[23] ^= 0x7F; + Files.write(f, bytes); + + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + // Only the intact prefix [s0, s1] is trusted; the corrupt e2 + // and everything after it are dropped. No recovered symbol is + // the corrupted string -- the tear is DETECTED, never silently + // misattributed. + Assert.assertEquals("parse must stop at the corrupt interior entry", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("s0", s.getQuick(0)); + Assert.assertEquals("s1", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { // C1 regression: the write path caps nothing, so a symbol larger than the From 6cc1307e4b1421e422be8f853c612d4059a4d7d0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 21:48:52 +0100 Subject: [PATCH 36/78] Document split-flush and dict-getter contracts Two documentation-only additions to the QWP store-and-forward client, making previously implicit contracts explicit. No behavior change. flushPendingRowsSplit: add a "Not atomic across frames" note. A sealAndSwapBuffer() failure on split frame k>1 leaves frames 1..k-1 published as deferred; the throw skips the trailing table-buffer reset, so the next flush re-emits the whole batch and the eventual commit commits the already-published prefix twice. This is pre-existing and within store-and-forward's at-least-once contract (absorbed by a DEDUP table or a durable-ack await); the note names the atomic-split fix (roll back or skip the published prefix on retry) as the larger follow-up. PersistedSymbolDict: mark loadedEntriesAddr(), loadedEntriesLen() and readLoadedSymbols() as construction-phase only. They read the native entry region that close() frees and nulls, with no closed-guard, so they are safe only before the slot's I/O thread and any producer append start; reading them from a running thread would risk a use-after-free. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cutlass/qwp/client/QwpWebSocketSender.java | 16 ++++++++++++++++ .../client/sf/cursor/PersistedSymbolDict.java | 15 ++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d00ab872..39db7d41 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3526,6 +3526,22 @@ private void flushPendingRows(boolean deferCommit) { * own message. All messages except the last carry FLAG_DEFER_COMMIT * so the server appends rows without committing until the final * message arrives. + *

+ * Not atomic across frames. The frames publish one at a time, so a + * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on + * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout -- + * leaves frames 1..k-1 already on the ring as deferred (appended, not yet + * committed). The throw propagates past the {@code resetTableBuffersAfterFlush} + * at the end of the loop, so the source rows survive in their table buffers + * and the NEXT flush re-emits the whole batch; the eventual commit then + * commits the already-published prefix alongside the re-sent copies, + * delivering those rows at-least-once (duplicated), not exactly-once. This is + * within store-and-forward's at-least-once contract -- a DEDUP table or a + * durable-ack await absorbs the duplicate, and the symbol-dict state stays + * consistent on the retry (the re-sent frames carry empty deltas and the + * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic + * (rolling back the published prefix, or skipping it on retry) would be a + * larger change. * * @param deferCommit when true, ALL messages (including the last) * carry FLAG_DEFER_COMMIT. When false, only the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 6bdc8162..bb7884bd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -354,13 +354,22 @@ public synchronized void close() { * Base address of the loaded entry region -- the concatenated * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly * as a delta section carries them. Zero when nothing was recovered. + *

+ * Construction-phase only. This hands out a raw pointer into native + * memory that {@link #close()} frees and nulls, with no closed-guard and no + * synchronization. It is safe to read only BEFORE the slot's I/O thread and + * any producer append start -- i.e. while the send loop is being constructed + * or an orphan-drain is seeding its mirror, both of which happen-before those + * threads. A caller that reads it from a running thread races {@code close()} + * and can dereference freed memory (use-after-free). */ public long loadedEntriesAddr() { return loadedEntriesAddr; } /** - * Byte length of {@link #loadedEntriesAddr()}. + * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for + * the same reason -- see {@link #loadedEntriesAddr()}. */ public int loadedEntriesLen() { return loadedEntriesLen; @@ -371,6 +380,10 @@ public int loadedEntriesLen() { * (entry {@code i} is symbol id {@code i}). Used once on recovery to * repopulate the producer's global dictionary. Empty when nothing was * recovered. + *

+ * Construction-phase only -- like {@link #loadedEntriesAddr()}, this + * walks the native entry region {@link #close()} frees, with no closed-guard, + * so it must run before the I/O thread and any producer append start. */ public ObjList readLoadedSymbols() { ObjList out = new ObjList<>(Math.max(size, 1)); From c065ad40c97d9ad6844b17d92815124ebc1337ea Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 23:46:36 +0100 Subject: [PATCH 37/78] Stop split-flush stranding a deferred prefix flushPendingRowsSplit published each table's frame one at a time (deferred, uncommitted) and only checked a frame against the server batch cap after encoding it. An oversized non-first table therefore threw after an earlier table's frame was already on the ring, where a later commit delivered it as a partial batch -- while resetTableBuffersAfterFlush discarded every source row. The caller saw a failure for a batch that had partly committed. Pre-flight every split frame's size before publishing any of them, so the split is all-or-nothing: either every frame fits and all publish, or none publish and it throws with nothing stranded. simBaseline mirrors advanceSentMaxSymbolId, so a measured size equals the frame the publish loop builds; the pass is read-only on the table buffers and touches no delta/persist state. The cost is a second encode pass over the split batch, already the exceptional large-batch path. testOversizedTableSplitStrandsNothing reproduces the stranding -- it fails on the old code with the earlier frame reaching the server -- and passes with the pre-flight. Also add testReconnectPreservesMonotonicDeltaBaseline: it pins that the producer's sent-symbol watermark survives a reconnect, asserting the first post-reconnect data frame carries a delta above the baseline (deltaStart >= 1) rather than re-shipping the whole dictionary from 0. The existing catch-up tests assert only that the final dictionary is complete, which a reset-then-redefine also satisfies. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 55 +++++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 75 ++++++++++++++++++- .../qwp/client/SelfSufficientFramesTest.java | 59 +++++++++++++++ 3 files changed, 174 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 39db7d41..2fd785a6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3552,16 +3552,48 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); } - // Collect non-empty table indices so we know which is last. + // Collect non-empty table indices so we know which is last, AND pre-flight + // every split frame's size BEFORE publishing any of them. The split hands + // frames to the ring one at a time (all but the last deferred -- appended but + // uncommitted); if a later table's frame were only found oversized + // mid-publish, the already-published prefix would strand on the ring, a + // subsequent commit would deliver it as a partial batch, and + // resetTableBuffersAfterFlush would discard every source row -- a partial + // commit the caller was told (by the throw) had failed. Checking all sizes up + // front makes the split all-or-nothing: either every frame fits and all + // publish, or none publish and we throw with nothing stranded. The cost is a + // second encode pass over the split batch, which is already the exceptional + // large-batch path. encode is read-only on the table buffer, and simBaseline + // mirrors the publish loop's baseline advance (advanceSentMaxSymbolId), so + // each measured size equals the frame the publish loop will build; this pass + // mutates no delta/persist state (the defer-commit flag is a header bit that + // does not change frame size). int nonEmptyCount = 0; + int simBaseline = symbolDeltaBaseline(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { continue; } QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - nonEmptyCount++; + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + nonEmptyCount++; + encoder.beginMessage(1, globalSymbolDictionary, simBaseline, currentBatchMaxSymbolId); + encoder.addTable(tableBuffer); + int messageSize = encoder.finishMessage(); + if (messageSize > serverMaxBatchSize) { + resetTableBuffersAfterFlush(keys); + throw new LineSenderException("single table batch too large for server batch cap") + .put(" [table=").put(tableName) + .put(", messageSize=").put(messageSize) + .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); + } + // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's + // new ids, the remaining frames carry an empty delta above the baseline. + if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { + simBaseline = currentBatchMaxSymbolId; } } @@ -3590,14 +3622,15 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm encoder.addTable(tableBuffer); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - - if (messageSize > serverMaxBatchSize) { - resetTableBuffersAfterFlush(keys); - throw new LineSenderException("single table batch too large for server batch cap") - .put(" [table=").put(tableName) - .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); - } + // The pre-flight pass above already verified every split frame fits the + // cap, so none can be found oversized here -- which is what keeps this + // loop from publishing (and stranding) a deferred prefix before an + // oversized table. The assert guards a future divergence between the two + // passes; it deliberately does NOT reset+throw here, because by this + // point a prefix may already be on the ring. + assert messageSize <= serverMaxBatchSize + : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + serverMaxBatchSize + ']'; // Write-ahead persist before publish (see flushPendingRows). The // first split frame carries the batch's new symbols; the rest are diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index e7f27db1..0f133443 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -102,6 +102,52 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { }); } + @Test + public void testReconnectPreservesMonotonicDeltaBaseline() throws Exception { + // Regression: the producer's sent-symbol watermark (sentMaxSymbolId) must + // SURVIVE a reconnect. resetSymbolDictStateForNewConnection deliberately + // leaves it untouched -- the I/O thread re-registers the whole dictionary via + // a catch-up frame before replay, so the producer keeps shipping deltas ABOVE + // the baseline across the wire boundary. If a regression reset it on + // reconnect, the first post-reconnect data frame would re-ship the whole + // dictionary inline (deltaStart=0), pure wasted bandwidth. The sibling + // testReconnectCatchUpRebuildsDictionary asserts only that the final + // dictionary is complete -- which a reset-then-redefine ALSO satisfies (the + // server tolerates the redefinition) -- so it does NOT catch that regression. + // This pins the baseline survival directly: connection 2's data frame must + // carry a delta starting at id 1 (above alpha), not 0. + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Connection 1 registers alpha (id 0); the server ACKs and drops it. + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + waitFor(() -> handler.conn1Closed, 5_000); + + // Connection 2 (fresh) registers beta (id 1). With the baseline + // preserved, beta ships as a delta ABOVE id 0 (deltaStart=1). + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + Assert.assertTrue("connection 2 must re-register the dictionary via a catch-up first", + handler.sawZeroTableFrameOnConn2); + Assert.assertTrue("post-reconnect data frame must ship a delta ABOVE the surviving " + + "baseline (deltaStart >= 1); a reset baseline re-ships the whole " + + "dictionary from deltaStart 0", + handler.conn2SawDeltaAboveBaseline); + } + }); + } + @Test public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exception { // Regression (homogeneous single cap): a symbol whose length sits just below @@ -332,6 +378,13 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ // Set from the flags byte of the zero-table catch-up frame on connection 2: // the catch-up carries no rows and must defer its (empty) commit. volatile boolean catchUpDeferredOnConn2; + // Set when connection 2 receives a DATA frame (tableCount > 0) whose delta + // starts ABOVE id 0 (deltaStart >= 1). This can only happen if the producer's + // monotonic baseline SURVIVED the reconnect: a reset would re-ship the whole + // dictionary from deltaStart 0. Robust to replay -- a replayed pre-reconnect + // frame carries its original deltaStart 0, so only a genuinely-above-baseline + // post-reconnect frame trips it. + volatile boolean conn2SawDeltaAboveBaseline; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -356,10 +409,24 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); accumulate(data, dict); - if (connNumber == 2 && tableCount(data) == 0) { - sawZeroTableFrameOnConn2 = true; - // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). - catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; + if (connNumber == 2) { + if (tableCount(data) == 0) { + sawZeroTableFrameOnConn2 = true; + // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). + catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; + } else if (data.length >= 12 && (data[5] & 0x08) != 0) { + // A post-reconnect DATA frame carrying a delta section + // (FLAG_DELTA_SYMBOL_DICT = 0x08). A deltaStart >= 1 means the + // producer resumed the delta ABOVE the surviving baseline; a reset + // baseline would re-ship from deltaStart 0. Checking any frame (not + // just the first) keeps this robust to a replayed pre-reconnect + // frame arriving ahead of the new one -- that replay carries its + // original deltaStart 0 and does not trip the flag. + int[] pos = {12}; + if (readVarint(data, pos) >= 1) { + conn2SawDeltaAboveBaseline = true; + } + } } try { client.sendBinary(buildAck(nextSeq.getAndIncrement())); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index cb2b0118..9277addd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; import org.junit.Test; @@ -273,6 +274,64 @@ public void testSplitBatchShipsDeltaOnFirstFrameOnly() throws Exception { }); } + @Test + public void testOversizedTableSplitStrandsNothing() throws Exception { + // Regression: flushPendingRowsSplit publishes each table's frame one at a + // time (all but the last deferred, i.e. appended but uncommitted). If a LATER + // table's frame exceeds the cap, the split must not have already published an + // earlier table's frame -- otherwise that prefix strands on the ring, a later + // commit delivers it as a partial batch, and resetTableBuffersAfterFlush + // discards every source row, all while flush() reported failure to the + // caller. The pre-flight size pass makes the split all-or-nothing: an + // oversized table throws BEFORE any frame is published. Pre-fix, the "small" + // table's frame was published and committed on close, so the server saw it. + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(200); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // auto_flush_bytes=off lets "big" accumulate PAST the cap (byte-based + // auto-flush is otherwise clamped to 90% of the cap and would flush + // first); the row/interval limits are set high so nothing auto-flushes + // during the test. Each row stays under the per-row guard (< cap), but + // 12 rows make "big"'s single frame exceed the cap, which no split can + // shrink. "small" (added first) fits; "big" (added second) does not, so + // the split hits it AFTER publishing "small" pre-fix. + String pad = new String(new char[40]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) { + sender.table("small").symbol("s", "a").longColumn("v", 1L).atNow(); + for (int i = 0; i < 12; i++) { + sender.table("big").stringColumn("p", pad).longColumn("v", (long) i).atNow(); + } + try { + sender.flush(); + Assert.fail("an oversized single-table split frame must throw"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), + e.getMessage().contains("too large for server batch cap")); + } + // close() drains the ring: pre-fix, the stranded "small" frame + // would be sent (and committed) here. + } + + // No DATA frame (tableCount > 0) may have reached the server: the + // oversized-table split published nothing. Pre-fix, "small" arrived. + long dataFrames = 0; + for (byte[] frame : handler.batches) { + if (frame.length >= 8 && (((frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8)) > 0)) { + dataFrames++; + } + } + Assert.assertEquals("an oversized-table split must publish NO data frame -- an " + + "earlier table's frame must not strand on the ring", 0, dataFrames); + } + }); + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; From 40c6a28d4ece5d71157ca8dfd6c4c10cb07f7f2e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 11 Jul 2026 00:05:32 +0100 Subject: [PATCH 38/78] Harden persisted symbol-dict edge paths Fold in low-severity hardening found in review; none changes behavior on any reachable input. PersistedSymbolDict: - appendRawEntries fails loudly (rather than flushing an uninitialised scratch tail and mis-advancing size) if a caller ever passes an (addr, len, count) triple whose entries do not fill len; the sole caller derives both from one beginMessage, so this cannot fire today. - openExisting sets entriesLen alongside the malloc, so the catch frees the right size if the copy walk throws -- the comment already claimed this held. - ensureScratch grows in long so scratchCap*2 cannot overflow negative past ~1 GB, matching ensureSentDictCapacity. - Two varint decode loops gain the shift > 35 bound the other decoders already carry. Close a file-descriptor leak in the rmDir test helpers: Files.walk returns a Stream backed by an open directory handle, so wrap it in try-with-resources (PersistedSymbolDictTest calls rmDir 13 times). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 36 ++++++++++++++++--- .../qwp/client/SelfSufficientFramesTest.java | 22 +++++++----- ...CursorWebSocketSendLoopMirrorLeakTest.java | 22 +++++++----- .../sf/cursor/PersistedSymbolDictTest.java | 20 ++++++----- 4 files changed, 70 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index bb7884bd..3d32aa65 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -230,6 +230,12 @@ public synchronized void appendRawEntries(long addr, int len, int count) { break; } shift += 7; + if (shift > 35) { + // A canonical entry-length varint is <= 5 bytes; a longer + // continuation run is corrupt. The downstream entryEnd > srcLimit + // check then rejects it. Matches decodeVarint / readVarintAt. + break; + } } long entryEnd = src + symLen; // src is just past the len varint if (entryEnd > srcLimit) { @@ -242,6 +248,16 @@ public synchronized void appendRawEntries(long addr, int len, int count) { dst += wireSpan + CRC_SIZE; src = entryEnd; } + if (src != srcLimit) { + // The count entries did not consume exactly len bytes -- a caller passed an + // inconsistent (addr, len, count) triple. Writing outLen would flush an + // uninitialised scratch tail and mis-advance size, so fail loudly. The sole + // caller derives count and len from one beginMessage, so this cannot fire + // today. + throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " + + FILE_NAME + " [count=" + count + ", len=" + len + + ", consumed=" + (int) (src - addr) + ']'); + } long written = Files.write(fd, scratchAddr, outLen, appendOffset); if (written != outLen) { throw new IllegalStateException("short write to " + FILE_NAME @@ -503,6 +519,9 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { // open. A second no-alloc walk over the already-validated region. if (wireLen > 0) { entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT); + // Record the length alongside the malloc so the catch below frees the + // right size (not 0) if this copy walk ever throws. + entriesLen = wireLen; long dst = entriesAddr; int p = HEADER_SIZE; for (int i = 0; i < count; i++) { @@ -516,6 +535,9 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { break; } shift += 7; + if (shift > 35) { + break; // corrupt run; these entries were CRC-validated above + } } int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan); @@ -523,7 +545,6 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { p += wireSpan + CRC_SIZE; // skip the entry's CRC } } - entriesLen = wireLen; Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // Drop any torn/stale trailing bytes so a LATER, shorter append cannot @@ -601,8 +622,15 @@ private void ensureScratch(int required) { if (scratchCap >= required) { return; } - int newCap = Math.max(required, Math.max(256, scratchCap * 2)); - scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, newCap, MemoryTag.NATIVE_DEFAULT); - scratchCap = newCap; + // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and + // would make the realloc size negative. required is bounded by one frame's + // entries (the server batch cap), so this never actually caps -- it mirrors the + // long-math growth in CursorWebSocketSendLoop.ensureSentDictCapacity. + long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2)); + if (newCap > Integer.MAX_VALUE - 8) { + newCap = Integer.MAX_VALUE - 8; + } + scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT); + scratchCap = (int) newCap; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 9277addd..812a5317 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -38,6 +38,7 @@ import java.util.Comparator; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -351,15 +352,18 @@ private static void rmDir(Path dir) { if (dir == null || !Files.exists(dir)) { return; } - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); + // try-with-resources: Files.walk returns a Stream backed by an open + // directory handle that must be closed, or each rmDir leaks a descriptor. + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } } catch (IOException ignored) { // best-effort } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 057e1a5f..38741039 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -38,6 +38,7 @@ import java.nio.file.Path; import java.util.Comparator; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -199,15 +200,18 @@ private static void rmDir(Path dir) throws IOException { if (dir == null || !Files.exists(dir)) { return; } - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); + // try-with-resources: Files.walk returns a Stream backed by an open + // directory handle that must be closed, or each rmDir leaks a descriptor. + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } } private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index cfcf4cf5..79a9a221 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -37,6 +37,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Comparator; +import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -503,14 +504,17 @@ private static void rmDir(Path dir) { if (dir == null || !Files.exists(dir)) { return; } - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - } - }); + // try-with-resources: Files.walk returns a Stream backed by an open + // directory handle that must be closed, or each rmDir leaks a descriptor. + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + } + }); + } } catch (IOException ignored) { } } From 94e3b1d2a9907e38d20ddc3f06ae702132f83891 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 11 Jul 2026 22:50:50 +0100 Subject: [PATCH 39/78] Guard the symbol-dict reopen against a >2GB file open() cast the on-disk file length to int before reading the whole file into one native buffer. A .symbol-dict that grew past 2GB (an extreme ~100M+ distinct symbols on one slot) therefore reopened with a negative malloc, a malloc(0) followed by a 4-byte out-of-bounds read, or a small-positive prefix whose validLen --- .../client/sf/cursor/PersistedSymbolDict.java | 94 ++++++--- .../sf/cursor/PersistedSymbolDictTest.java | 186 ++++++++++++++++++ 2 files changed, 254 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 3d32aa65..1d545f10 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -27,7 +27,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Crc32c; -import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; @@ -120,6 +120,10 @@ public final class PersistedSymbolDict implements QuietCloseable { static final byte VERSION = 2; // v2 appended the per-entry CRC-32C private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; + // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files); + // tests inject a fault facade to exercise recovery I/O failures (a truncate + // that cannot drop a torn tail, a short write) without a real broken disk. + private final FilesFacade ff; private long appendOffset; private boolean closed; // In-memory copy of the entry region [len][utf8]... exactly as on disk, @@ -136,7 +140,8 @@ public final class PersistedSymbolDict implements QuietCloseable { private int scratchCap; private int size; - private PersistedSymbolDict(int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { + private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { + this.ff = ff; this.fd = fd; this.appendOffset = appendOffset; this.size = size; @@ -153,17 +158,40 @@ private PersistedSymbolDict(int fd, long appendOffset, int size, long loadedEntr * so a broken side-file degrades gracefully rather than aborting the sender. */ public static PersistedSymbolDict open(String slotDir) { + return open(FilesFacade.INSTANCE, slotDir); + } + + /** + * Facade-aware variant of {@link #open(String)}. Production passes + * {@link FilesFacade#INSTANCE}; tests inject a fault facade to drive recovery + * I/O failures (e.g. a truncate that cannot drop a torn tail). + */ + public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; - long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; + // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be + // reopened: openExisting reads it into ONE int-sized native buffer, and + // the (int) cast of a >2GB length is either negative (malloc rejects it), + // exactly zero (getInt then reads 4 bytes past a zero-size allocation), or + // a small positive prefix (whose validLen < len branch would then + // DESTRUCTIVELY truncate the multi-GB file). Recreate empty instead -- + // fail-clean, exactly like every other unreadable-file case here, so the + // sender falls back to full self-sufficient frames. Reaching this needs + // ~100M+ distinct symbols on one slot (far past realistic symbol + // cardinality); the guard keeps the read/write size boundary safe anyway. + if (existing > Integer.MAX_VALUE) { + LOG.warn("symbol dict {} too large ({} bytes) to reopen; recreating empty", filePath, existing); + return openFresh(ff, filePath); + } if (existing >= HEADER_SIZE) { - PersistedSymbolDict d = openExisting(filePath, existing); + PersistedSymbolDict d = openExisting(ff, filePath, existing); if (d != null) { return d; } // Fall through to a clean re-create: a header/parse failure on an // existing file means it cannot be trusted for delta replay. } - return openFresh(filePath); + return openFresh(ff, filePath); } /** @@ -183,7 +211,14 @@ public static PersistedSymbolDict open(String slotDir) { * does. */ public static PersistedSymbolDict openClean(String slotDir) { - return openFresh(slotDir + "/" + FILE_NAME); + return openClean(FilesFacade.INSTANCE, slotDir); + } + + /** + * Facade-aware variant of {@link #openClean(String)}. + */ + public static PersistedSymbolDict openClean(FilesFacade ff, String slotDir) { + return openFresh(ff, slotDir + "/" + FILE_NAME); } /** @@ -194,7 +229,14 @@ public static PersistedSymbolDict openClean(String slotDir) { * failed delete cannot leave a stale dictionary a new session would trust. */ public static void removeOrphan(String slotDir) { - Files.remove(slotDir + "/" + FILE_NAME); + removeOrphan(FilesFacade.INSTANCE, slotDir); + } + + /** + * Facade-aware variant of {@link #removeOrphan(String)}. + */ + public static void removeOrphan(FilesFacade ff, String slotDir) { + ff.remove(slotDir + "/" + FILE_NAME); } /** @@ -258,7 +300,7 @@ public synchronized void appendRawEntries(long addr, int len, int count) { + FILE_NAME + " [count=" + count + ", len=" + len + ", consumed=" + (int) (src - addr) + ']'); } - long written = Files.write(fd, scratchAddr, outLen, appendOffset); + long written = ff.write(fd, scratchAddr, outLen, appendOffset); if (written != outLen) { throw new IllegalStateException("short write to " + FILE_NAME + " [expected=" + outLen + ", actual=" + written + ']'); @@ -290,7 +332,7 @@ public synchronized void appendSymbol(CharSequence symbol) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen)); - long written = Files.write(fd, scratchAddr, recLen, appendOffset); + long written = ff.write(fd, scratchAddr, recLen, appendOffset); if (written != recLen) { throw new IllegalStateException("short write to " + FILE_NAME + " [expected=" + recLen + ", actual=" + written + ']'); @@ -333,7 +375,7 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen)); len += wireLen + CRC_SIZE; } - long written = Files.write(fd, scratchAddr, len, appendOffset); + long written = ff.write(fd, scratchAddr, len, appendOffset); if (written != len) { throw new IllegalStateException("short write to " + FILE_NAME + " [expected=" + len + ", actual=" + written + ']'); @@ -362,7 +404,7 @@ public synchronized void close() { scratchCap = 0; } if (fd >= 0) { - Files.close(fd); + ff.close(fd); } } @@ -456,8 +498,8 @@ private static long[] decodeVarint(long buf, int pos, int limit) { return null; } - private static PersistedSymbolDict openExisting(String filePath, long fileLen) { - int fd = Files.openRW(filePath); + private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen) { + int fd = ff.openRW(filePath); if (fd < 0) { LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd); return null; @@ -468,13 +510,13 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { try { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - long read = Files.read(fd, buf, len, 0); + long read = ff.read(fd, buf, len, 0); if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); - Files.close(fd); + ff.close(fd); return null; } // Parse complete, CRC-valid entries after the header; stop at the first @@ -555,15 +597,15 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { // open() then recreates it empty (fail-clean) rather than risk a silent // misattribution. long validLen = HEADER_SIZE + diskConsumed; - if (validLen < len && !Files.truncate(fd, validLen)) { + if (validLen < len && !ff.truncate(fd, validLen)) { LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } - Files.close(fd); + ff.close(fd); return null; } - return new PersistedSymbolDict(fd, validLen, count, entriesAddr, entriesLen); + return new PersistedSymbolDict(ff, fd, validLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); @@ -575,14 +617,14 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } - Files.close(fd); + ff.close(fd); LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); return null; } } - private static PersistedSymbolDict openFresh(String filePath) { - int fd = Files.openCleanRW(filePath); + private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { + int fd = ff.openCleanRW(filePath); if (fd < 0) { LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); return null; @@ -595,10 +637,10 @@ private static PersistedSymbolDict openFresh(String filePath) { Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0); Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0); - long written = Files.write(fd, hdr, HEADER_SIZE, 0); + long written = ff.write(fd, hdr, HEADER_SIZE, 0); if (written != HEADER_SIZE) { - Files.close(fd); - Files.remove(filePath); + ff.close(fd); + ff.remove(filePath); LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; } @@ -607,7 +649,7 @@ private static PersistedSymbolDict openFresh(String filePath) { // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte // malloc cannot realistically OOM), but close the fd against a future // edit so it cannot leak -- mirroring openExisting's error handling. - Files.close(fd); + ff.close(fd); LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); return null; } finally { @@ -615,7 +657,7 @@ private static PersistedSymbolDict openFresh(String filePath) { Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); } } - return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); + return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0); } private void ensureScratch(int required) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 79a9a221..966cdffd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -499,6 +500,53 @@ public void testTornTrailingEntrySelfHeals() throws Exception { }); } + @Test + public void testTruncateFailureRecreatesEmpty() throws Exception { + // A host crash can leave a torn/stale tail past the last complete entry. + // open() drops it with a truncate; if that truncate FAILS (a read-only + // remount, a Windows share lock), the file still exposes the stale bytes, + // whose self-consistent per-entry CRC a later shifted parse could accept as + // a real symbol. So a failed truncate must make the file UNTRUSTED -- + // open() recreates it empty (fail-clean) rather than returning a dict laid + // over stale bytes. Drive the truncate failure with a facade and assert the + // reopened dictionary is empty, not the [one, two] prefix. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("one"); + d.appendSymbol("two"); + d.close(); + + // Append a torn trailing record (length prefix 5, only 2 bytes) so + // the reopen parses [one, two], then finds validLen < len and tries + // to truncate the tail -- the branch under test. + Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + + // Reopen through a facade whose truncate() fails. + PersistedSymbolDict re = PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString()); + Assert.assertNotNull(re); + try { + // The failed truncate made the file untrusted, so open() recreated + // it empty rather than trusting the [one, two] prefix over a stale + // tail: size()==0, not 2, and no recovered symbols. + Assert.assertEquals("failed truncate must recreate the dictionary empty", 0, re.size()); + Assert.assertEquals(0, re.readLoadedSymbols().size()); + // openFresh rewrote a bare 8-byte header (magic + version), so both + // the two entries and the torn tail are gone. + Assert.assertEquals("recreated file is a bare header", 8L, Files.size(f)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + private static void rmDir(Path dir) { try { if (dir == null || !Files.exists(dir)) { @@ -518,4 +566,142 @@ private static void rmDir(Path dir) { } catch (IOException ignored) { } } + + /** + * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE} + * except {@link #truncate(int, long)}, which always fails -- reproducing a + * host where the torn/stale-tail truncate cannot succeed (read-only remount, + * Windows share lock) so {@code open()}'s fail-clean recreate path runs. + */ + private static final class FailingTruncateFacade implements FilesFacade { + @Override + public long allocNativePath(String path) { + return INSTANCE.allocNativePath(path); + } + + @Override + public boolean allocate(int fd, long size) { + return INSTANCE.allocate(fd, size); + } + + @Override + public int close(int fd) { + return INSTANCE.close(fd); + } + + @Override + public boolean exists(String path) { + return INSTANCE.exists(path); + } + + @Override + public void findClose(long findPtr) { + INSTANCE.findClose(findPtr); + } + + @Override + public long findFirst(String dir) { + return INSTANCE.findFirst(dir); + } + + @Override + public long findName(long findPtr) { + return INSTANCE.findName(findPtr); + } + + @Override + public int findNext(long findPtr) { + return INSTANCE.findNext(findPtr); + } + + @Override + public int findType(long findPtr) { + return INSTANCE.findType(findPtr); + } + + @Override + public void freeNativePath(long pathPtr) { + INSTANCE.freeNativePath(pathPtr); + } + + @Override + public int fsync(int fd) { + return INSTANCE.fsync(fd); + } + + @Override + public long length(int fd) { + return INSTANCE.length(fd); + } + + @Override + public long length(String path) { + return INSTANCE.length(path); + } + + @Override + public long length(long pathPtr) { + return INSTANCE.length(pathPtr); + } + + @Override + public int lock(int fd) { + return INSTANCE.lock(fd); + } + + @Override + public int mkdir(String path, int mode) { + return INSTANCE.mkdir(path, mode); + } + + @Override + public int openCleanRW(String path) { + return INSTANCE.openCleanRW(path); + } + + @Override + public int openCleanRW(long pathPtr) { + return INSTANCE.openCleanRW(pathPtr); + } + + @Override + public int openRW(String path) { + return INSTANCE.openRW(path); + } + + @Override + public int openRW(long pathPtr) { + return INSTANCE.openRW(pathPtr); + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + + @Override + public boolean remove(String path) { + return INSTANCE.remove(path); + } + + @Override + public boolean remove(long pathPtr) { + return INSTANCE.remove(pathPtr); + } + + @Override + public int rename(String oldPath, String newPath) { + return INSTANCE.rename(oldPath, newPath); + } + + @Override + public boolean truncate(int fd, long size) { + return false; // the fault under test + } + + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } } From 6c6dc40d6000c561e81dbda8bab0e61628a7dc8f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 11 Jul 2026 23:02:12 +0100 Subject: [PATCH 40/78] Latch a terminal on sent-dict mirror overflow ensureSentDictCapacity threw a bare LineSenderException when the lifetime-monotonic sent-dictionary mirror would exceed MAX_SENT_DICT_BYTES. accumulateSentDict runs after the frame's sendBinary, so that throw unwound to ioLoop -> fail() -> connectLoop with running still true, which reconnected and replayed the same frame; the mirror never shrinks, so the replay re-overflowed and the loop reconnected again -- an unbounded livelock instead of the loud failure the ceiling promises. recordFatal now flips running=false before the throw, so connectLoop's !running guard winds the loop down and checkError() surfaces the terminal to the producer. This matches sendCatchUpChunk's guard for the same ceiling. The throw stays, to unwind past the pending copyMemory. The path needs ~100M+ distinct symbols on one connection to reach, so it is defensive; the producer's GlobalSymbolDictionary would OOM first. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorWebSocketSendLoop.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index a855cc50..a24776d3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2137,8 +2137,19 @@ private void ensureSentDictCapacity(long required) { return; } if (required > MAX_SENT_DICT_BYTES) { - throw new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + // Latch a terminal, do NOT just throw: accumulateSentDict runs AFTER + // the frame's sendBinary, so a bare throw unwinds to ioLoop -> fail() + // -> connectLoop, which (running still true) reconnects and replays the + // same frame, which re-overflows the never-shrinking mirror -- an + // unbounded reconnect livelock rather than the "fails loudly" the + // MAX_SENT_DICT_BYTES ceiling promises. recordFatal flips running=false + // so connectLoop's !running guard winds the loop down and checkError() + // surfaces the terminal, matching sendCatchUpChunk's guard for the same + // ceiling. The throw still unwinds past the pending copyMemory. + LineSenderException err = new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw err; } // Grow in long to avoid the capacity*2 int overflow (negative) that would // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. From ff2e841af2e2d10e9568737806dd119f075be17f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 00:16:03 +0100 Subject: [PATCH 41/78] Guard torn-dict resume and skip split re-encode Two hardening fixes to the delta symbol-dictionary path. Torn-dictionary resume (data integrity): the persisted .symbol-dict is not fsync'd, so a host crash can lose its newest entries while the segment frames that introduced those ids survive and replay. A resumed producer, seeded from the shorter dictionary, would reuse ids the surviving frames already define and silently misattribute symbols. The send loop's replay guard only catches a gap (deltaStart > mirror); the contiguous tear self-heals the mirror and leaves the producer diverged. CursorSendEngine now records the highest symbol id the recovered frames reference via a read-only MmapSegment/SegmentRing walk, and seedGlobalDictionaryFromPersisted fails clean when it reaches the recovered dictionary size -- the affected data must be resent. Only the resuming producer refuses; the orphan drainer still self-heals and drains the slot. testTornDictSubsetFailsCleanOnResume covers it. Split-flush sizing (performance): flushPendingRowsSplit re-encoded the whole batch a second time just to size each split frame. It now derives each frame size arithmetically from the combined encode flushPendingRows already performed, capturing each table's body bytes and the delta-entry length. Body encoding is context-free, so the sizes are exact; the publish loop's assert cross-checks them. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 81 ++++++++++++++++--- .../client/sf/cursor/CursorSendEngine.java | 31 +++++++ .../qwp/client/sf/cursor/MmapSegment.java | 66 +++++++++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 18 +++++ .../qwp/client/DeltaDictRecoveryTest.java | 69 ++++++++++++++++ 5 files changed, 253 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 2fd785a6..1f948a72 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -57,6 +57,7 @@ import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.IntList; import io.questdb.client.std.Misc; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -169,6 +170,11 @@ public class QwpWebSocketSender implements Sender { // behind a drainer's endpoint walk. private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; + // Per-table encoded body byte counts captured during flushPendingRows' combined + // encode, reused by flushPendingRowsSplit to size each split frame arithmetically + // instead of re-encoding the batch a second time. Cleared and repopulated on every + // flush; only consumed on the (exceptional) split path. Reused to stay zero-GC. + private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -3474,6 +3480,15 @@ private void flushPendingRows(boolean deferCommit) { encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, symbolDeltaBaseline(), currentBatchMaxSymbolId); + // Record each table's encoded body size (position delta across addTable) so + // the split path can size its per-table frames arithmetically rather than + // re-encoding the whole batch. Body encoding is context-free (a table's bytes + // don't depend on its siblings or the delta section), so the size a table + // takes here equals the size it takes in its own split frame. Only consumed + // when the batch overflows the cap; the capture is a couple of int ops per + // table on the common path. + splitFrameBodyBytes.clear(); + int bodyStart = encoder.getBuffer().getPosition(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3490,12 +3505,18 @@ private void flushPendingRows(boolean deferCommit) { } encoder.addTable(tableBuffer); + int bodyEnd = encoder.getBuffer().getPosition(); + splitFrameBodyBytes.add(bodyEnd - bodyStart); + bodyStart = bodyEnd; } int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { - flushPendingRowsSplit(keys, deferCommit); + // The combined frame's delta-entry bytes are byte-identical to the first + // split frame's (same baseline + batch max), so capture the length now + // for the arithmetic frame-sizing in flushPendingRowsSplit. + flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen()); return; } @@ -3547,7 +3568,7 @@ private void flushPendingRows(boolean deferCommit) { * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) { + private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen) { if (LOG.isDebugEnabled()) { LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); } @@ -3561,15 +3582,19 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // resetTableBuffersAfterFlush would discard every source row -- a partial // commit the caller was told (by the throw) had failed. Checking all sizes up // front makes the split all-or-nothing: either every frame fits and all - // publish, or none publish and we throw with nothing stranded. The cost is a - // second encode pass over the split batch, which is already the exceptional - // large-batch path. encode is read-only on the table buffer, and simBaseline - // mirrors the publish loop's baseline advance (advanceSentMaxSymbolId), so - // each measured size equals the frame the publish loop will build; this pass - // mutates no delta/persist state (the defer-commit flag is a header bit that - // does not change frame size). + // publish, or none publish and we throw with nothing stranded. + // + // Each split frame's size is derived ARITHMETICALLY from the combined encode + // flushPendingRows already performed -- header + the two delta-section varints + // + the delta entries (byte-identical to the combined frame's when this frame + // carries them) + the table's own body bytes (captured in splitFrameBodyBytes, + // context-free so identical here and in its solo frame) -- rather than + // re-encoding every table a second time. simBaseline mirrors the publish loop's + // baseline advance (advanceSentMaxSymbolId), so each size equals the frame the + // publish loop will build; this pass mutates no delta/persist state. int nonEmptyCount = 0; int simBaseline = symbolDeltaBaseline(); + int bodyIdx = 0; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3580,9 +3605,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm continue; } nonEmptyCount++; - encoder.beginMessage(1, globalSymbolDictionary, simBaseline, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); + int deltaStart = simBaseline + 1; + int deltaCount = Math.max(0, currentBatchMaxSymbolId - simBaseline); + int messageSize = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + (deltaCount > 0 ? combinedDeltaEntriesLen : 0) + + splitFrameBodyBytes.getQuick(bodyIdx); + bodyIdx++; if (messageSize > serverMaxBatchSize) { resetTableBuffersAfterFlush(keys); throw new LineSenderException("single table batch too large for server batch cap") @@ -3821,11 +3851,38 @@ private void resetSymbolDictStateForNewConnection() { * dictionary shorter than {@code pd.size()} and desyncing * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()}, * which silently misattributes later symbols after a reconnect. + *

+ * Host-crash tear guard. The persisted dictionary is NOT fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can lose its + * most-recently-written (highest-id) entries while the segment frames that + * introduced those ids survive -- and those newest frames, being the least + * likely to be acked, replay on recovery. The send loop's catch-up mirror then + * rebuilds the missing ids from those frames' own delta bytes, but THIS producer + * -- seeded only from the shorter dictionary -- would assign its next new symbol + * an id the surviving frames already define, putting two symbols on one id and + * silently misattributing values. The send loop's replay guard only catches a + * GAP ({@code deltaStart > sentDictCount}); a frame that introduces exactly the + * torn-off id ({@code deltaStart == pd.size()}) slips through and self-heals the + * mirror, leaving only this producer diverged. Detect it here -- the surviving + * frames reference an id at or beyond the recovered dictionary size -- and fail + * clean: the affected data must be resent, matching the design's torn-dict + * "resend required" contract. The recovered data is not lost; the background + * drainer still drains this slot (its mirror self-heals from the frames). Only a + * host crash reaches this -- a process crash keeps the page cache, so the + * write-ahead ordering keeps the dictionary a superset of the frames. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { if (pd == null || pd.size() == 0) { return; } + if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { + throw new LineSenderException( + "recovered store-and-forward symbol dictionary is a subset of the surviving frames " + + "(likely a host crash tore its unsynced tail): frames reference symbol id " + + cursorEngine.recoveredMaxSymbolId() + " but the recovered dictionary holds only " + + pd.size() + " id(s); resuming would reuse ids the frames already define -- " + + "resend the affected data"); + } ObjList symbols = pd.readLoadedSymbols(); for (int i = 0, n = symbols.size(); i < n; i++) { globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 6c34124d..eb59633a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -96,6 +96,15 @@ public final class CursorSendEngine implements QuietCloseable { // covers them. Read by the sender's close-time drain to avoid waiting on // acks that cannot arrive. private long recoveredCommitBoundaryFsn = -1L; + // Highest symbol id any recovered delta frame references, or -1 for + // fresh/memory rings (and recovered rings with no symbol-bearing frame). A + // resuming producer seeds its dictionary baseline from the persisted + // .symbol-dict; if that dictionary was torn below this id by a host crash + // (the side-file is not fsync'd), the producer would re-use ids the surviving + // frames already define. seedGlobalDictionaryFromPersisted compares this + // against the recovered dictionary size to fail clean instead. Computed once + // in the constructor's recovery branch; -1 elsewhere. + private long recoveredMaxSymbolId = -1L; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -312,6 +321,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } + // Highest symbol id the surviving frames reference. A resuming + // producer compares this against its recovered dictionary size + // (seedGlobalDictionaryFromPersisted) to detect a host-crash tear: + // if a frame references an id the (unsynced, torn) .symbol-dict no + // longer holds, resuming would re-use it. maxSymbolDeltaEnd returns + // 0 when no frame carries a symbol, yielding -1 here. Computed + // before the I/O loop or producer append; single-threaded here. + this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( + io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, + io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, + io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT, + io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE) - 1L; } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers @@ -714,6 +735,16 @@ public long recoveredCommitBoundaryFsn() { return recoveredCommitBoundaryFsn; } + /** + * Highest symbol id any recovered delta frame references, or {@code -1} for + * fresh/memory rings (and recovered rings with no symbol-bearing frame). A + * resuming producer compares this against its recovered dictionary size to + * detect a host-crash tear of the persisted {@code .symbol-dict}. + */ + public long recoveredMaxSymbolId() { + return recoveredMaxSymbolId; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..04f2ac93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -522,6 +522,72 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in return best; } + /** + * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) that + * any symbol-dict delta frame in this segment references, or {@code 0} when no + * such frame carries a symbol. Read-only walk over the recovered frames, used + * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, + * out-of-order page loss) below the ids the surviving frames still reference: if + * the max here reaches at or beyond the recovered dictionary size, a resuming + * producer -- seeded from the shorter dictionary -- would re-use ids the frames + * already define. The frame layout mirrors {@link #findLastFrameFsnWithoutPayloadFlag}: + * the QWP message header ({@code qwpHeaderSize} bytes) is followed by the delta + * section {@code [deltaStart varint][deltaCount varint]...}. + * + * @param headerMagic the QWP message magic identifying a well-formed frame + * @param flagsOffset byte offset of the flags field within the QWP header + * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit + * @param qwpHeaderSize the QWP message header size (delta section starts past it) + */ + public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + long maxEnd = 0L; + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + long payload = mmapAddress + off + FRAME_HEADER_SIZE; + if (payloadLen >= qwpHeaderSize + && payloadLen > flagsOffset + && Unsafe.getUnsafe().getInt(payload) == headerMagic + && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { + long p = payload + qwpHeaderSize; + long limit = payload + payloadLen; + long deltaStart = 0L; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaStart |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; // corrupt run; recovered frames are CRC-valid, so defensive only + } + } + long deltaCount = 0L; + shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaCount |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; + } + } + long end = deltaStart + deltaCount; + if (end > maxEnd) { + maxEnd = end; + } + } + off += FRAME_HEADER_SIZE + payloadLen; + } + return maxEnd; + } + /** * Number of frames written since {@link #create} (or recovered by * {@link #openExisting}). Used by {@code SegmentRing} to compute diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index c8516c4c..2b53e956 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -546,6 +546,24 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag return Math.max(best, fsn); } + /** + * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame in the + * ring references (0 when none). See {@link MmapSegment#maxSymbolDeltaEnd}; + * used once at recovery to detect a persisted dictionary torn below the ids + * the surviving frames reference. + */ + public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + long maxEnd = 0L; + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + if (end > maxEnd) { + maxEnd = end; + } + } + long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + return Math.max(maxEnd, end); + } + public MmapSegment getActive() { return active; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e46d0148..21023c04 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -771,6 +771,75 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti }); } + @Test + public void testTornDictSubsetFailsCleanOnResume() throws Exception { + // C1 regression: the persisted .symbol-dict is NOT fsync'd, so a host crash + // can lose its highest-id tail entries while the segment frames that + // introduced those ids survive (out-of-order page loss). Recovery then seeds + // the producer from the SHORTER dictionary, but the I/O-thread catch-up mirror + // rebuilds the missing ids from those surviving frames -- so a resumed producer + // would assign its next new symbol an id the frames already define, silently + // misattributing symbol values on the wire. The send loop's replay guard only + // catches a GAP (deltaStart > mirror); a frame introducing exactly the + // torn-off id slips through and self-heals the mirror, leaving only this + // producer diverged. seedGlobalDictionaryFromPersisted must detect that the + // surviving frames reference an id at/beyond the recovered dictionary size and + // fail clean. Without the fix the resume succeeds and the reuse corrupts + // silently -- so the build below would NOT throw and the test's fail() fires. + assertMemoryLeak(() -> { + // Phase 1: record three delta frames (a@0, b@1, c@2) to disk with nothing + // acked (silent server), so all three survive and replay on recovery. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + Sender s1 = Sender.fromConfig(cfg); + try { + s1.table("m").symbol("s", "a").longColumn("v", 0).atNow(); + s1.flush(); + s1.table("m").symbol("s", "b").longColumn("v", 1).atNow(); + s1.flush(); + s1.table("m").symbol("s", "c").longColumn("v", 2).atNow(); + s1.flush(); + } finally { + s1.close(); + } + } + + // Simulate the host-crash tear: rewrite the persisted dictionary to drop + // its highest entry (c@2), keeping a@0,b@1 -- while the frame referencing + // c@2 stays on disk. openClean truncates; the two appends leave the exact + // two-entry dictionary a torn tail would recover to. + String slotDir = Paths.get(sfDir, "default").toString(); + try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { + Assert.assertNotNull(torn); + torn.appendSymbol("a"); + torn.appendSymbol("b"); + Assert.assertEquals(2, torn.size()); + } + + // Phase 2: a resuming sender must refuse -- the surviving frames reference + // id 2 but the recovered dictionary holds only 2 id(s) [0,1]. + try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try { + Sender resumed = Sender.fromConfig(cfg); + resumed.close(); + Assert.fail("resume on a torn (subset) dictionary must fail clean"); + } catch (LineSenderException expected) { + Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), + expected.getMessage().contains("subset of the surviving frames") + && expected.getMessage().contains("resend")); + } + } + }); + } + private static int globalDictSize(Sender sender) throws Exception { Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); f.setAccessible(true); From 50fc21814f24eca96514fcea7d220431aeb6bd9d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 00:37:08 +0100 Subject: [PATCH 42/78] Give the catch-up cap gap a settle budget Two hardening fixes to the split/catch-up paths for heterogeneous / rolling-cap clusters. Catch-up cap-gap settle budget (M1): when a symbol-dict catch-up entry is too large for the fresh server's advertised batch cap, sendDictCatchUp latched a terminal on first sight. A homogeneous cluster never trips this (an entry that fit its data frame under a cap always fits its bare catch-up frame under that same cap), but a failover to a smaller-cap node could kill the sender for an entry an earlier node accepted. It now retries across MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive reconnects (throwing a retriable CatchUpSendException, no recordFatal), riding out the transient window until a larger-cap node returns; only a persistent gap latches the terminal. A successful catch-up resets the budget, and a transient reconnect never reaches the catch-up so it neither increments nor burns it. Matches the orphan drainer's durable-ack settle budget. Split-flush cap snapshot (M2): flushPendingRows now snapshots the volatile serverMaxBatchSize once and threads it into flushPendingRowsSplit for both the pre-flight sizing and the publish-loop assert. A mid-flush failover (the I/O thread lowers the cap) could previously make the two passes size against different caps, breaking the all-or-nothing guarantee and firing the assert on a legitimate race. The next flush picks up the new cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 32 ++++++---- .../sf/cursor/CursorWebSocketSendLoop.java | 63 ++++++++++++++----- .../qwp/client/DeltaDictCatchUpTest.java | 17 +++-- ...WebSocketSendLoopCatchUpAlignmentTest.java | 56 +++++++++++++++++ 4 files changed, 136 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1f948a72..3765ce87 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3512,11 +3512,19 @@ private void flushPendingRows(boolean deferCommit) { int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { + // Snapshot the volatile cap ONCE for this whole flush. The I/O thread lowers + // serverMaxBatchSize on a mid-stream failover to a smaller-cap node + // (applyServerBatchSizeLimit); if the split pre-flight and the publish loop + // re-read the field independently, a failover between them would size frames + // against two different caps -- breaking the all-or-nothing guarantee and + // firing the publish-loop assert on a legitimate race. Both use this snapshot; + // the next flush picks up the new cap. + int cap = serverMaxBatchSize; + if (cap > 0 && messageSize > cap) { // The combined frame's delta-entry bytes are byte-identical to the first // split frame's (same baseline + batch max), so capture the length now // for the arithmetic frame-sizing in flushPendingRowsSplit. - flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen()); + flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen(), cap); return; } @@ -3568,9 +3576,9 @@ private void flushPendingRows(boolean deferCommit) { * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen) { + private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen, int cap) { if (LOG.isDebugEnabled()) { - LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); + LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); } // Collect non-empty table indices so we know which is last, AND pre-flight @@ -3613,12 +3621,12 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm + (deltaCount > 0 ? combinedDeltaEntriesLen : 0) + splitFrameBodyBytes.getQuick(bodyIdx); bodyIdx++; - if (messageSize > serverMaxBatchSize) { + if (messageSize > cap) { resetTableBuffersAfterFlush(keys); throw new LineSenderException("single table batch too large for server batch cap") .put(" [table=").put(tableName) .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); + .put(", serverMaxBatchSize=").put(cap).put(']'); } // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's // new ids, the remaining frames carry an empty delta above the baseline. @@ -3655,12 +3663,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // The pre-flight pass above already verified every split frame fits the // cap, so none can be found oversized here -- which is what keeps this // loop from publishing (and stranding) a deferred prefix before an - // oversized table. The assert guards a future divergence between the two - // passes; it deliberately does NOT reset+throw here, because by this - // point a prefix may already be on the ring. - assert messageSize <= serverMaxBatchSize + // oversized table. Both passes size against the SAME snapshot cap, so a + // mid-flush failover cannot make them disagree; the assert therefore only + // catches a genuine divergence between the pre-flight arithmetic and the + // real encode (a future bug), not a cap race. It deliberately does NOT + // reset+throw here, because by this point a prefix may already be on the ring. + assert messageSize <= cap : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName - + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + serverMaxBatchSize + ']'; + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; // Write-ahead persist before publish (see flushPendingRows). The // first split frame carries the batch's new symbols; the rest are diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index a24776d3..1b4a7d6f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -133,6 +133,18 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Settle budget for the symbol-dict catch-up cap gap: how many CONSECUTIVE + // reconnect attempts may find a single dictionary entry too large for the fresh + // server's advertised batch cap before the sender latches a terminal. A + // homogeneous cluster never trips it -- an entry that fit its data frame under a + // cap always fits its bare catch-up frame under that same cap -- so this only + // affects a heterogeneous / rolling-cap cluster, where a failover to a + // smaller-cap node can hit it for an entry an earlier node accepted. Retrying + // rides out the transient window until a larger-cap node returns; only a + // persistent gap (every reachable node too small for this many attempts) latches + // terminal, matching the orphan drainer's DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS. + // A successful catch-up resets the counter (see sendDictCatchUp). + private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even // this needs ~200M+ distinct symbols on a single connection, far past any real @@ -206,6 +218,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // for the connection's lifetime (a reconnect may need the whole dictionary at // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; + // Consecutive reconnect attempts whose symbol-dict catch-up found an entry too + // large for the fresh server's batch cap (see MAX_CATCHUP_CAP_GAP_ATTEMPTS). A + // successful catch-up resets it; it is NOT reset per connection -- it measures + // the cap-gap episode across reconnects so a persistent gap eventually latches. + // I/O-thread-only. + private int catchUpCapGapAttempts; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies @@ -2243,24 +2261,36 @@ private int sendDictCatchUp() { + NativeBufferWriter.varintSize(1) + entryBytes; if (soloFrameLen > frameLimit) { - // Non-retriable: the entry will not shrink and the same cluster - // re-advertises the same cap, so reconnecting would livelock. - // Latch a terminal (the data must be resent after the cap is - // raised) rather than calling fail() -- which, from inside the - // catch-up, would re-enter connectLoop (see CatchUpSendException). + // Cap gap: this entry cannot be re-registered under the fresh + // server's advertised cap. A HOMOGENEOUS cluster never reaches here + // (an entry that fit its data frame under a cap always fits its bare + // catch-up frame under that same cap), so the only way in is a + // heterogeneous / rolling-cap failover to a smaller-cap node. // - // Tradeoff (heterogeneous / rolling-cap clusters): a symbol - // accepted under a larger/absent cap can hit this on failover to a - // smaller-cap node, and the hard terminal does NOT self-recover if - // a later node advertises a larger cap -- the producer must be - // resumed after the data is resent (or the cap raised). Bounding - // symbol size at ingest, or a settle budget across reconnects - // before latching, would relax this, but both are larger changes; - // the terminal keeps the homogeneous common case livelock-free. + // Give the cluster a settle budget instead of latching on first + // sight: a larger-cap node may return, so retry across reconnects + // and only latch a terminal after MAX_CATCHUP_CAP_GAP_ATTEMPTS + // consecutive attempts still find it too large. Under budget the + // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with + // backoff and re-runs the catch-up, which resets the counter on a + // node that accepts it. A transient reconnect (connect/upgrade + // failure, role reject) never reaches the catch-up, so it neither + // increments nor burns this budget. On exhaustion latch via + // recordFatal, NOT fail() -- failing from inside the catch-up would + // re-enter connectLoop (see CatchUpSendException); the data must be + // resent after the cap is raised. + catchUpCapGapAttempts++; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS; LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "frameLen=" + soloFrameLen + ", cap=" + cap + ']'); - recordFatal(err); + + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + ']' + + (exhausted + ? "; the data must be resent after the cap is raised" + : "; retrying -- a larger-cap node may return")); + if (exhausted) { + recordFatal(err); + } throw new CatchUpSendException(err); } if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { @@ -2279,6 +2309,9 @@ private int sendDictCatchUp() { sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; } + // The whole dictionary re-registered without a cap gap: this node accepts + // every entry, so the cap-gap episode (if any) is over -- reset the budget. + catchUpCapGapAttempts = 0; return framesSent; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 0f133443..132f2c06 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -212,11 +212,15 @@ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exce @Test public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // A dictionary entry that exceeds the reconnect server's per-chunk budget - // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. - // sendDictCatchUp must latch a clean terminal ("... during catch-up") - // rather than call fail(): pre-fix the oversized entry drove an endless - // reconnect loop (the entry never shrinks and the same cluster - // re-advertises the same cap) and re-entered connectLoop from the catch-up. + // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. Every + // reachable node re-advertises the same small cap here, so the gap never + // resolves: sendDictCatchUp must retry across the settle budget and then + // latch a clean terminal ("... during catch-up") rather than call fail() + // (which from inside the catch-up re-enters connectLoop). Pre-fix it latched + // on the FIRST cap gap; the settle budget (MAX_CATCHUP_CAP_GAP_ATTEMPTS) + // rides out a transient smaller-cap window first (see the retry sibling in + // CursorWebSocketSendLoopCatchUpAlignmentTest), and only a persistent gap + // exhausts it. Small reconnect backoffs keep the budgeted attempts fast. // // Connection 1 advertises no cap, so the ~202-byte symbol registers and // enters the sent-dictionary mirror. The handler then shrinks the @@ -234,7 +238,8 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry LineSenderException terminal = null; - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;"); try { sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); sender.flush(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 3814d826..315e3d75 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -26,6 +26,7 @@ import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; @@ -265,6 +266,61 @@ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { }); } + @Test + public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { + // M1: an entry too large for the fresh server's cap during catch-up (a + // heterogeneous / rolling-cap failover to a smaller-cap node) must NOT latch + // on first sight. sendDictCatchUp throws a RETRIABLE CatchUpSendException so + // the reconnect loop rides it out -- a larger-cap node may return -- and only + // after MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive cap gaps does it recordFatal. + // Pre-fix the first cap gap latched a terminal, so one transient failover to a + // smaller-cap node killed the sender. (A successful catch-up resets the budget; + // the other catch-up tests, which use a fitting cap, never trip it.) + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol. + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + // Attempts 1 .. max-1 are retriable: no terminal is latched. + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + assertTrue("attempt " + i + " must name the catch-up cap gap: " + + e.getCause().getMessage(), + e.getCause().getMessage().contains("during catch-up")); + } + loop.checkError(); // under budget => retriable => no terminal + } + // The exhausting attempt still throws, and now latches the terminal. + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("the exhausting cap gap must still raise CatchUpSendException"); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + try { + loop.checkError(); + fail("exhausting the cap-gap settle budget must latch a terminal"); + } catch (LineSenderException terminal) { + assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up") + && terminal.getMessage().contains("must be resent")); + } + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { From 7880b7a2730304a42c79ba37ef15e4dcca4951f9 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 14:13:24 +0100 Subject: [PATCH 43/78] Cover defensive dict guards and tidy test helpers Round of minor cleanups on the delta symbol-dictionary work. Tests for previously-uncovered defensive guards (all proven to fail without their guard): PersistedSymbolDict.open recreates empty on a bad version byte (a valid entry behind a corrupted version must not parse) and on a > Integer.MAX_VALUE length (a >2GB dictionary can't map into one int buffer -- driven by a fault facade); and the send loop's ensureSentDictCapacity latches a terminal instead of overflowing the int capacity math when the mirror would exceed MAX_SENT_DICT_BYTES. openExisting nulls buf after freeing it on the bad-magic/bad-version path, so the catch block cannot double-free if ff.close ever threw. PersistedSymbolDict.appendSymbol is marked @TestOnly -- production persists a frame's whole new-symbol range via appendSymbols / appendRawEntries; only tests append one symbol at a time. Documented that symbol-dict catch-up frames sit outside the poison-frame detector: a deterministically-NACKed catch-up recycles forever (paced), which is fine since a catch-up only re-registers already-accepted symbols. Consolidated four reinvented recursive rmDir/rmDirRec test helpers into one TestUtils.removeTmpDirRec (the existing removeTmpDir is flat and can't clean the store-and-forward slot subdirectory layout), and removed the imports that freed up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 7 + .../client/sf/cursor/PersistedSymbolDict.java | 7 + .../qwp/client/DeltaDictRecoveryTest.java | 30 +---- .../qwp/client/SelfSufficientFramesTest.java | 23 +--- ...WebSocketSendLoopCatchUpAlignmentTest.java | 41 ++++++ ...CursorWebSocketSendLoopMirrorLeakTest.java | 19 +-- .../sf/cursor/PersistedSymbolDictTest.java | 124 ++++++++++++++---- .../questdb/client/test/tools/TestUtils.java | 30 +++++ 8 files changed, 188 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 1b4a7d6f..f55f47c8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2911,6 +2911,13 @@ private void handleServerRejection(long wireSeq) { // path, where engine.acknowledge() no-ops at or below ackedFsn. A // real replayed data frame is at fsn > ackedFsn, so it is never // caught here. + // + // Catch-up frames therefore sit OUTSIDE the poison detector: a + // deterministically-NACKed catch-up recycles forever (paced, so no + // busy-loop). That is acceptable -- a catch-up only re-registers + // symbols the cluster already accepted, so a persistent NACK of one + // is a server bug, not a poison-frame the client can quarantine, and + // Invariant B's "retry a transient outage forever" applies. handlePreSendRejection(wireSeq, status, category, policy); return; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 1d545f10..9930a239 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -33,6 +33,7 @@ import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import io.questdb.client.std.str.Utf8s; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -317,7 +318,12 @@ public synchronized void appendRawEntries(long addr, int len, int count) { * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on * recovery. + *

+ * Test-only: production persists a frame's whole new-symbol range in one write + * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to + * build a dictionary one entry at a time. */ + @TestOnly public synchronized void appendSymbol(CharSequence symbol) { if (closed) { return; @@ -516,6 +522,7 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + buf = 0L; // null after free so the catch below cannot double-free if ff.close throws ff.close(fd); return null; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 21023c04..c7d4de13 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -27,7 +27,6 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; -import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -76,9 +75,7 @@ public void setUp() { @After public void tearDown() { - if (sfDir != null) { - rmDirRec(sfDir); - } + TestUtils.removeTmpDirRec(sfDir); } @Test @@ -879,31 +876,6 @@ private static int readVarint(byte[] buf, int[] pos) { throw new IllegalStateException("varint truncated"); } - private static void rmDirRec(String dir) { - if (!Files.exists(dir)) { - return; - } - long find = Files.findFirst(dir); - if (find > 0) { - try { - int rc = 1; - while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - if (name != null && !".".equals(name) && !"..".equals(name)) { - String child = dir + "/" + name; - if (!Files.remove(child)) { - rmDirRec(child); - } - } - rc = Files.findNext(find); - } - } finally { - Files.findClose(find); - } - } - Files.remove(dir); - } - /** * Reconstructs the per-connection symbol dictionary from delta sections, * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 812a5317..524063a1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -27,6 +27,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -35,10 +36,8 @@ import java.nio.ByteOrder; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Comparator; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -348,25 +347,7 @@ private static int readVarint(byte[] buf, int offset) { } private static void rmDir(Path dir) { - try { - if (dir == null || !Files.exists(dir)) { - return; - } - // try-with-resources: Files.walk returns a Stream backed by an open - // directory handle that must be closed, or each rmDir leaks a descriptor. - try (Stream walk = Files.walk(dir)) { - walk.sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); - } - } catch (IOException ignored) { - // best-effort - } + TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } private static void waitFor(BoolCondition cond, long timeoutMillis) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 315e3d75..54fd645c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -321,6 +321,47 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { }); } + @Test + public void testMirrorOverflowFailsLoud() throws Exception { + // ensureSentDictCapacity must latch a terminal -- not silently overflow the + // int capacity math into a heap-corrupting copyMemory -- when the sent-dict + // mirror would exceed MAX_SENT_DICT_BYTES. Unreachable at real cardinality + // (~200M+ symbols on one connection), so drive the guard directly with an + // oversized required, mirroring testCatchUpChunkFrameSizeOverflowFailsLoud. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_SENT_DICT_BYTES"); + maxField.setAccessible(true); + long overCeiling = (long) maxField.getInt(null) + 1L; + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("ensureSentDictCapacity", long.class); + m.setAccessible(true); + try { + m.invoke(loop, overCeiling); + fail("a mirror capacity past MAX_SENT_DICT_BYTES must fail loud, not overflow"); + } catch (InvocationTargetException e) { + assertEquals("overflow must surface as LineSenderException", + "LineSenderException", e.getCause().getClass().getSimpleName()); + assertTrue("message must name the mirror ceiling: " + e.getCause().getMessage(), + e.getCause().getMessage().contains("mirror exceeds the maximum size")); + } + // recordFatal (not a bare throw) latched the terminal, so the loop + // winds down instead of reconnecting into the same overflow. + try { + loop.checkError(); + fail("mirror overflow must latch a terminal"); + } catch (LineSenderException terminal) { + assertTrue(terminal.getMessage().contains("mirror exceeds the maximum size")); + } + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 38741039..5cff81ba 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -36,9 +37,7 @@ import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Comparator; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -197,21 +196,7 @@ private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exc } private static void rmDir(Path dir) throws IOException { - if (dir == null || !Files.exists(dir)) { - return; - } - // try-with-resources: Files.walk returns a Stream backed by an open - // directory handle that must be closed, or each rmDir leaks a descriptor. - try (Stream walk = Files.walk(dir)) { - walk.sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); - } + TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 966cdffd..eb79e016 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -32,13 +32,10 @@ import org.junit.Assert; import org.junit.Test; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Comparator; -import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -230,6 +227,43 @@ public void testBadMagicIsRecreatedEmpty() throws Exception { }); } + @Test + public void testBadVersionIsRecreatedEmpty() throws Exception { + // A file with correct 'SYD1' magic and a VALID entry but an unknown version + // byte must be recreated empty -- its entries belong to a foreign/future + // format and must not be parsed as v2. Covers the version sub-condition + // specifically (testBadMagicIsRecreatedEmpty covers the magic one). Because + // the seeded "a" is a valid v2 entry, without the version check it would parse + // back (size 1), so this fails. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + Path f = dir.resolve(".symbol-dict"); + // Build a real v2 dictionary with one valid entry... + PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(seed); + seed.appendSymbol("a"); + seed.close(); + // ...then corrupt ONLY the version byte (offset 4) to an unknown value. + byte[] bytes = Files.readAllBytes(f); + bytes[4] = (byte) 99; + Files.write(f, bytes); + + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals("bad-version file recreated empty (entries must not parse)", 0, d.size()); + d.appendSymbol("X"); + Assert.assertEquals(1, d.size()); + } finally { + d.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testCloseNullsLoadedEntries() throws Exception { // close() must null loadedEntriesAddr/Len after freeing them (like @@ -450,6 +484,38 @@ public void testRemoveOrphanDeletesFile() throws Exception { }); } + @Test + public void testTooLargeToReopenRecreatesEmpty() throws Exception { + // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be read + // into one int-sized buffer; open() must recreate it empty (fail-clean, like + // every other unreadable-file case) rather than truncate the multi-GB file + // via a negative/zero (int) length cast. A fault facade reports the huge + // length without needing a real 2GB file on disk. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Seed a small real file so exists() is true and the facade's huge + // length() drives the > Integer.MAX_VALUE reopen guard. + PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(seed); + seed.appendSymbol("a"); + seed.close(); + + PersistedSymbolDict d = PersistedSymbolDict.open(new HugeLengthFacade(), dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals("a >2GB dictionary must recreate empty", 0, d.size()); + d.appendSymbol("X"); + Assert.assertEquals(1, d.size()); + } finally { + d.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testTornTrailingEntrySelfHeals() throws Exception { assertMemoryLeak(() -> { @@ -548,32 +614,14 @@ public void testTruncateFailureRecreatesEmpty() throws Exception { } private static void rmDir(Path dir) { - try { - if (dir == null || !Files.exists(dir)) { - return; - } - // try-with-resources: Files.walk returns a Stream backed by an open - // directory handle that must be closed, or each rmDir leaks a descriptor. - try (Stream walk = Files.walk(dir)) { - walk.sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - } - }); - } - } catch (IOException ignored) { - } + TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } /** - * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE} - * except {@link #truncate(int, long)}, which always fails -- reproducing a - * host where the torn/stale-tail truncate cannot succeed (read-only remount, - * Windows share lock) so {@code open()}'s fail-clean recreate path runs. + * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE}. + * Subclasses inject a single fault; every other call hits the real filesystem. */ - private static final class FailingTruncateFacade implements FilesFacade { + private static class DelegatingFilesFacade implements FilesFacade { @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); @@ -696,7 +744,7 @@ public int rename(String oldPath, String newPath) { @Override public boolean truncate(int fd, long size) { - return false; // the fault under test + return INSTANCE.truncate(fd, size); } @Override @@ -704,4 +752,28 @@ public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } } + + /** + * Fails every {@link #truncate(int, long)} -- reproducing a host where the + * torn/stale-tail truncate cannot succeed (read-only remount, Windows share + * lock) so {@code open()}'s fail-clean recreate path runs. + */ + private static final class FailingTruncateFacade extends DelegatingFilesFacade { + @Override + public boolean truncate(int fd, long size) { + return false; + } + } + + /** + * Reports a dictionary length past {@link Integer#MAX_VALUE} -- reproducing a + * dictionary that legitimately grew beyond 2GB, which {@code open()} cannot read + * into one int-sized buffer and must recreate empty. + */ + private static final class HugeLengthFacade extends DelegatingFilesFacade { + @Override + public long length(String path) { + return (long) Integer.MAX_VALUE + 1L; + } + } } diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java index 7572880b..39755426 100644 --- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java @@ -215,6 +215,36 @@ public static void removeTmpDir(String tmpDir) { Files.remove(tmpDir); } + /** + * Recursive counterpart to {@link #removeTmpDir(String)} for tests whose temp + * directory has subdirectories -- e.g. the store-and-forward slot layout + * {@code

/default/...}, which the flat variant cannot clean up. A + * {@code null} argument is a no-op, so it is safe from {@code tearDown} before + * {@code setUp} ran. Uses {@code java.nio.file} (fully qualified to avoid the + * {@code io.questdb.client.std.Files} import clash) so subdirectories delete + * bottom-up. + */ + public static void removeTmpDirRec(String tmpDir) { + if (tmpDir == null) { + return; + } + java.nio.file.Path root = java.nio.file.Paths.get(tmpDir); + if (!java.nio.file.Files.exists(root)) { + return; + } + // try-with-resources: the walk Stream holds an open directory handle that + // must be closed, or each call leaks a descriptor. + try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) { + walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + java.nio.file.Files.deleteIfExists(p); + } catch (java.io.IOException ignored) { + } + }); + } catch (java.io.IOException ignored) { + } + } + /** * Java 8 stand-in for {@code String.repeat(int)} (added in Java 11). */ From 2ad8f5018788549dd75ba64a8d4249032f8a6d5b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:09:58 +0100 Subject: [PATCH 44/78] Fail clean on a total symbol-dict tear on resume seedGlobalDictionaryFromPersisted returned early when the persisted .symbol-dict recovered empty (size 0), skipping the torn-dictionary guard on the line just below. A host crash -- or a prior full-dict-mode session whose openClean failed and never wrote the side-file -- can leave a size-0 dictionary while the segment frames that introduced its ids survive. Resuming then seeds the producer with an empty dictionary, so the next new symbol reuses id 0; once that frame is trimmed and the connection recycles, the catch-up re-registers the recovered ids and the producer's rows are silently misattributed. The send loop's replay guard cannot catch this: a deltaStart=0 frame self-heals the catch-up mirror rather than tripping the gap check, so the seed-time guard is the only defense. Run it before the size-0 short-circuit. A genuinely empty slot (no symbol-bearing frames) still has recoveredMaxSymbolId == -1, so -1 >= 0 is false and the empty case returns without throwing. Add testTornDictTotalLossFailsCleanOnResume covering the size-0 total tear (frames replay from deltaStart=0, so only the seed-time guard can reject it). Repoint two tests that had reached the I/O-thread guard through the bypassed early return: testTornDictionaryFailsCleanly... now expects the build-time guard, and testTornDictionaryOneIdGap... reaches the I/O guard through the unopenable-dictionary path instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 14 +- .../qwp/client/DeltaDictRecoveryTest.java | 146 ++++++++++++------ 2 files changed, 116 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3765ce87..4f7c374b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3882,9 +3882,18 @@ private void resetSymbolDictStateForNewConnection() { * write-ahead ordering keeps the dictionary a superset of the frames. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { - if (pd == null || pd.size() == 0) { + if (pd == null) { return; } + // Run the torn-dictionary guard BEFORE the empty-dictionary short-circuit + // below: a TOTAL tear (pd.size() == 0) with surviving symbol-bearing frames + // must fail clean too, not slip through. Such a frame starts at deltaStart=0 + // and self-heals the I/O-thread catch-up mirror, so the send loop's replay + // guard (deltaStart > sentDictCount) never fires -- this seed-time guard is + // then the only defense against the producer resuming unseeded and silently + // reusing ids the frames already define. A genuinely empty slot (no + // symbol-bearing frames) has recoveredMaxSymbolId() == -1, so -1 >= 0 is false + // and the pd.size() == 0 return below still fires. if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { throw new LineSenderException( "recovered store-and-forward symbol dictionary is a subset of the surviving frames " @@ -3893,6 +3902,9 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + pd.size() + " id(s); resuming would reuse ids the frames already define -- " + "resend the affected data"); } + if (pd.size() == 0) { + return; + } ObjList symbols = pd.readLoadedSymbols(); for (int i = 0, n = symbols.size(); i < n; i++) { globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index c7d4de13..fb355b82 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -171,19 +171,21 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception } } - // Simulate a host/power crash: the segment frames survive but the persisted - // dictionary is lost, and the ack watermark was left mid-stream. Truncate - // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at - // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + // Simulate a host/power crash: the segment frames survive but the WHOLE + // persisted dictionary is lost (truncate .symbol-dict to its 8-byte header, + // 0 symbols), and the ack watermark was left mid-stream (FSN 2). The + // surviving frames still reference the lost ids. java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); java.nio.file.Files.write(dict, header); writeAckWatermark(slot.resolve(".ack-watermark"), 2); - // Phase 2: recover against a fresh counting server. The replay guard must - // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally - // rather than send a gapped frame that would corrupt the table. + // Phase 2: recover against a fresh counting server. With the whole + // dictionary lost (size 0) while the surviving frames still reference its + // ids, seedGlobalDictionaryFromPersisted must refuse the resume at BUILD -- + // its size==0 short-circuit must NOT skip the torn-dict guard -- rather than + // let the producer resume unseeded and silently misattribute reused ids. CountingHandler handler = new CountingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); @@ -192,47 +194,34 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); try { - // Poll for the replay guard to fire (it recordFatal's on the I/O - // thread); flush() surfaces the latched terminal to the producer. - // A bounded poll replaces a fixed sleep and captures it as soon as - // it fires; close() below is the fallback if it surfaces only there. - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - s2.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - s2.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } - } + Sender s2 = Sender.fromConfig(cfg); + s2.close(); + } catch (LineSenderException e) { + terminal = e; } Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", 0, handler.frames.get()); Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("symbol dictionary is incomplete")); + terminal.getMessage().contains("subset of the surviving frames") + && terminal.getMessage().contains("resend")); } }); } @Test public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { - // Boundary variant of testTornDictionaryFailsCleanlyInsteadOfCorrupting: the - // recovered dictionary is short by EXACTLY ONE id -- the tightest gap the - // guard must still reject (deltaStart == recoveredSize + 1). A one-entry-short - // tail is the common host-crash outcome, so this pins the guard's - // false-negative edge: a "deltaStart > size + 1" mutation would let this - // single-id gap through and null-pad the missing symbol on the server. + // One-id-gap boundary variant of + // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames: the first + // replayed frame starts EXACTLY ONE id past the empty mirror -- the tightest + // gap the I/O-thread replay guard must still reject (deltaStart == + // sentDictCount + 1). It reaches that guard via the unopenable-dictionary path + // (deltaDictEnabled=false), NOT a size-0 openable dict, which now fails clean + // earlier in seedGlobalDictionaryFromPersisted. A one-entry-short tail is the + // common host-crash outcome, so this pins the guard's false-negative edge: a + // "deltaStart > size + 1" mutation would let this single-id gap through and + // null-pad the missing symbol on the server. assertMemoryLeak(() -> { // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i). try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { @@ -249,15 +238,17 @@ public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { } } - // Lose the whole dictionary (truncate to the 8-byte header, size 0) but - // stamp the watermark at FSN 0, so recovery replays from FSN 1 -- a frame - // with deltaStart=1. The dictionary covers no ids, so id 0 is the single - // missing entry: deltaStart(1) == recoveredSize(0) + 1. Because size is 0 - // no catch-up is sent, so any counted frame would be the gapped data frame. + // Make the persisted dictionary UNOPENABLE (replace .symbol-dict with a + // directory, so both openRW and openCleanRW fail and the engine reports + // deltaDictEnabled=false), then stamp the watermark at FSN 0 so recovery + // replays from FSN 1 -- a frame with deltaStart=1. The mirror is unseeded + // (size 0), so id 0 is the single missing entry: deltaStart(1) == + // sentDictCount(0) + 1. No catch-up is sent, so any counted frame would be + // the gapped data frame. java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); - byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); - java.nio.file.Files.write(dict, header); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); writeAckWatermark(slot.resolve(".ack-watermark"), 0); // Phase 2: recover against a fresh counting server. The guard must fire on @@ -837,6 +828,75 @@ public void testTornDictSubsetFailsCleanOnResume() throws Exception { }); } + @Test + public void testTornDictTotalLossFailsCleanOnResume() throws Exception { + // C1 regression (TOTAL-loss variant of testTornDictSubsetFailsCleanOnResume): + // a host crash can lose the ENTIRE persisted .symbol-dict (size 0) while the + // frames that introduced its ids survive; equivalently, a prior session whose + // openClean failed ran full-dict mode and never wrote the side-file, so this + // session recovers those frames against a FRESH EMPTY dictionary. The recovered + // frames start at deltaStart=0 and self-heal the I/O-thread catch-up mirror, so + // the send loop's replay guard (deltaStart > mirror) never fires -- only + // seedGlobalDictionaryFromPersisted can catch it. Its pd.size()==0 early return + // must NOT skip the torn-dict guard: the surviving frames reference id 2 while + // the recovered dictionary holds 0 id(s), so resuming unseeded would reuse those + // ids and silently misattribute symbol values on the wire. Without the fix the + // build below does NOT throw (the guard is bypassed) and the test's fail() fires. + assertMemoryLeak(() -> { + // Phase 1: record three delta frames (a@0, b@1, c@2) with nothing acked, so + // all three survive and replay -- from FSN 0 (deltaStart=0), the case the + // I/O-thread guard self-heals rather than rejects. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + Sender s1 = Sender.fromConfig(cfg); + try { + s1.table("m").symbol("s", "a").longColumn("v", 0).atNow(); + s1.flush(); + s1.table("m").symbol("s", "b").longColumn("v", 1).atNow(); + s1.flush(); + s1.table("m").symbol("s", "c").longColumn("v", 2).atNow(); + s1.flush(); + } finally { + s1.close(); + } + } + + // Total dictionary loss: openClean truncates .symbol-dict to its bare header + // (size 0) with zero appends, while every frame -- including the ones + // referencing a@0,b@1,c@2 -- stays on disk. This is the fresh-empty-dict + // state a full-dict-mode prior session or a total host-crash tear leaves. + String slotDir = Paths.get(sfDir, "default").toString(); + try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { + Assert.assertNotNull(torn); + Assert.assertEquals(0, torn.size()); + } + + // Phase 2: a resuming sender must refuse at build -- the surviving frames + // reference id 2 but the recovered dictionary holds 0 id(s). The I/O-thread + // guard cannot help (deltaStart 0 self-heals the mirror), so the seed-time + // guard is the sole defense. + try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try { + Sender resumed = Sender.fromConfig(cfg); + resumed.close(); + Assert.fail("resume on a totally lost (size 0) dictionary must fail clean"); + } catch (LineSenderException expected) { + Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), + expected.getMessage().contains("subset of the surviving frames") + && expected.getMessage().contains("resend")); + } + } + }); + } + private static int globalDictSize(Sender sender) throws Exception { Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); f.setAccessible(true); From 76fe4f1d34b2ae5d49e863abbfc17952b09bf012 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:43:47 +0100 Subject: [PATCH 45/78] Bound recovered symbol id to committed frames CursorSendEngine.recoveredMaxSymbolId walked every recovered frame, including the aborted orphan-deferred tail -- the frames trySendOne retires without ever transmitting. Those frames typically introduce the highest symbol ids, so a host crash that tore the persisted .symbol-dict down to the committed ids while an orphan-tail frame referenced a higher one inflated recoveredMaxSymbolId and made seedGlobalDictionaryFromPersisted reject an otherwise fully recoverable slot. The producer never reuses an orphan id on the wire -- the tail retires before its new frames send -- so the rejection was a false positive. Bound the maxSymbolDeltaEnd walk to recoveredCommitBoundaryFsn so only committed frames count. MmapSegment and SegmentRing take a maxFsnInclusive argument; a frame whose fsn (baseSeq + i) exceeds it is skipped, and the walk still advances past it. A slot with no orphan tail is unchanged: the boundary equals publishedFsn, so every frame still counts. Add testRecoveredMaxSymbolIdExcludesOrphanTailFrames: a committed delta frame registering ids 0,1 plus a deferred (orphan) frame registering id 2 must recover recoveredMaxSymbolId == 1, not 2. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 20 ++++--- .../qwp/client/sf/cursor/MmapSegment.java | 29 ++++++---- .../qwp/client/sf/cursor/SegmentRing.java | 16 +++--- ...CursorWebSocketSendLoopOrphanTailTest.java | 53 +++++++++++++++++++ 4 files changed, 94 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index eb59633a..dde24142 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -321,18 +321,22 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } - // Highest symbol id the surviving frames reference. A resuming - // producer compares this against its recovered dictionary size - // (seedGlobalDictionaryFromPersisted) to detect a host-crash tear: - // if a frame references an id the (unsynced, torn) .symbol-dict no - // longer holds, resuming would re-use it. maxSymbolDeltaEnd returns - // 0 when no frame carries a symbol, yielding -1 here. Computed - // before the I/O loop or producer append; single-threaded here. + // Highest symbol id the surviving COMMITTED frames reference. A + // resuming producer compares this against its recovered dictionary + // size (seedGlobalDictionaryFromPersisted) to detect a host-crash + // tear: if a committed frame references an id the (unsynced, torn) + // .symbol-dict no longer holds, resuming would re-use it. The walk is + // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred + // tail -- retired without ever being transmitted -- does not inflate + // this and over-reject an otherwise-recoverable slot. maxSymbolDeltaEnd + // returns 0 when no such frame carries a symbol, yielding -1 here. + // Computed before the I/O loop or producer append; single-threaded. this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE) - 1L; + io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE, + recoveredCommitBoundaryFsn) - 1L; } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 04f2ac93..a7ee42ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -530,23 +530,34 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in * out-of-order page loss) below the ids the surviving frames still reference: if * the max here reaches at or beyond the recovered dictionary size, a resuming * producer -- seeded from the shorter dictionary -- would re-use ids the frames - * already define. The frame layout mirrors {@link #findLastFrameFsnWithoutPayloadFlag}: - * the QWP message header ({@code qwpHeaderSize} bytes) is followed by the delta - * section {@code [deltaStart varint][deltaCount varint]...}. + * already define. Only frames at or below {@code maxFsnInclusive} count: frames + * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires + * without ever transmitting, so their ids must not inflate the result (a resuming + * producer never reuses them on the wire). The frame layout mirrors + * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header + * ({@code qwpHeaderSize} bytes) is followed by the delta section + * {@code [deltaStart varint][deltaCount varint]...}. * - * @param headerMagic the QWP message magic identifying a well-formed frame - * @param flagsOffset byte offset of the flags field within the QWP header - * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit - * @param qwpHeaderSize the QWP message header size (delta section starts past it) + * @param headerMagic the QWP message magic identifying a well-formed frame + * @param flagsOffset byte offset of the flags field within the QWP header + * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit + * @param qwpHeaderSize the QWP message header size (delta section starts past it) + * @param maxFsnInclusive highest frame FSN to consider; frames above it are the + * retired orphan-deferred tail -- pass + * {@code recoveredCommitBoundaryFsn} */ - public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; long off = HEADER_SIZE; long frames = frameCount; for (long i = 0; i < frames; i++) { + long fsn = baseSeq + i; int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); long payload = mmapAddress + off + FRAME_HEADER_SIZE; - if (payloadLen >= qwpHeaderSize + // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never + // sent, so its ids must not count. off still advances below. + if (fsn <= maxFsnInclusive + && payloadLen >= qwpHeaderSize && payloadLen > flagsOffset && Unsafe.getUnsafe().getInt(payload) == headerMagic && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 2b53e956..21cfa9d1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -547,20 +547,22 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag } /** - * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame in the - * ring references (0 when none). See {@link MmapSegment#maxSymbolDeltaEnd}; - * used once at recovery to detect a persisted dictionary torn below the ids - * the surviving frames reference. + * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below + * {@code maxFsnInclusive} references (0 when none). See + * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a + * persisted dictionary torn below the ids the surviving committed frames + * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred + * tail and are excluded. */ - public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; for (int i = 0, n = sealedSegments.size(); i < n; i++) { - long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); if (end > maxEnd) { maxEnd = end; } } - long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); return Math.max(maxEnd, end); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index ff710fe1..ed3989ee 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -75,7 +75,9 @@ public class CursorWebSocketSendLoopOrphanTailTest { private static final int FLAG_DEFER_COMMIT = 0x01; + private static final int FLAG_DELTA_SYMBOL_DICT = 0x08; private static final int HEADER_OFFSET_FLAGS = 5; + private static final int HEADER_SIZE = 12; private static final int MAGIC_MESSAGE = 0x31505751; // "QWP1" little-endian private String tmpDir; @@ -221,6 +223,36 @@ public void testSlowPathReplaysBelowTailThenRetiresAndRecyclesOnce() throws Exce }); } + @Test + public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception { + // recoveredMaxSymbolId must reflect only COMMITTED (transmitted) frames, not + // the aborted orphan-tail frames trySendOne retires without ever sending. A + // host crash that tears the persisted dictionary down to the committed ids + // while an orphan-tail frame introduced a HIGHER id must NOT over-reject the + // resume: the producer never reuses an orphan id on the wire (the tail retires + // first), so counting it would inflate recoveredMaxSymbolId and make + // seedGlobalDictionaryFromPersisted fail a fully-recoverable slot. The + // maxSymbolDeltaEnd walk is therefore bounded to recoveredCommitBoundaryFsn. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + // fsn 0: commit-bearing delta frame registering ids 0,1. + appendDeltaFrame(engine, false, 0, 2); + // fsn 1: DEFERRED delta frame registering id 2 -- the orphan tail. + appendDeltaFrame(engine, true, 2, 1); + } + try (CursorSendEngine engine = newEngine()) { + assertTrue(engine.wasRecoveredFromDisk()); + assertEquals("last commit-bearing frame", 0L, engine.recoveredCommitBoundaryFsn()); + assertEquals("orphan tail tip", 1L, engine.recoveredOrphanTipFsn()); + // Only the committed frame's ids (0,1) count -> highest id 1. The + // orphan-tail frame's id 2 is excluded, so a resume whose recovered + // dictionary holds ids 0,1 (size 2) is NOT over-rejected. + assertEquals("orphan-tail id 2 must be excluded from recoveredMaxSymbolId", + 1L, engine.recoveredMaxSymbolId()); + } + }); + } + // --------------------------------------------------------------------- // harness // --------------------------------------------------------------------- @@ -291,6 +323,27 @@ private static void appendFrame(CursorSendEngine engine, boolean defer) { } } + // Appends a QWP frame carrying a symbol-dict delta section ([deltaStart varint] + // [deltaCount varint]) so the recovery walk's maxSymbolDeltaEnd counts it. + // deltaStart/deltaCount stay < 128 so each encodes in a single LEB128 byte. + private static void appendDeltaFrame(CursorSendEngine engine, boolean defer, int deltaStart, int deltaCount) { + int size = HEADER_SIZE + 2; + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < size; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS, + (byte) (FLAG_DELTA_SYMBOL_DICT | (defer ? FLAG_DEFER_COMMIT : 0))); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE, (byte) deltaStart); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 1, (byte) deltaCount); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException { long deadline = System.nanoTime() + 10_000_000_000L; while (engine.ackedFsn() < target) { From 084f93f23fdc0a5cf3109dd6d0aae0e06e3c4f3f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:44:01 +0100 Subject: [PATCH 46/78] Null loaded dict entries after a failed truncate PersistedSymbolDict.openExisting frees the loaded-entries buffer in the truncate-failure branch, then calls ff.close(fd). If that close threw, the enclosing catch re-freed the same buffer -- a double-free -- because, unlike the bad-magic branch, this branch did not null the pointer first. Null entriesAddr/entriesLen right after freeing them, matching the buf discipline above, so the catch's guard skips the second free. Correct the catch comment, which wrongly claimed nothing between the malloc and the return could throw -- the truncate branch's free-then-close is exactly such a point. This is defensive hardening: FilesFacade.close returns a status code and never throws (the native close never raises), so the path is unreachable today, but the nulling keeps it safe against a future facade or edit. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/sf/cursor/PersistedSymbolDict.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 9930a239..70cadb19 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -608,6 +608,10 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + // Null after freeing (like buf above) so the catch below cannot + // double-free entriesAddr if the following ff.close throws. + entriesAddr = 0L; + entriesLen = 0; } ff.close(fd); return null; @@ -617,10 +621,11 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); } - // entriesAddr is transferred to the returned dict on the success path, - // and the catch only runs before that return, so freeing it here cannot - // double-free. Unreachable today (nothing between its malloc and the - // return throws), but keeps the error path leak-free against a future edit. + // Free entriesAddr if it was allocated and not yet handed off. The success + // path transfers it to the returned dict, and every path that frees it + // earlier (the truncate-failure branch above) also nulls it, so this cannot + // double-free. Keeps the error path leak-free on any throw between its + // malloc and the return. if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } From 8facb8d2627f41cc51f206f68d901486f6988cfa Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:44:16 +0100 Subject: [PATCH 47/78] Test the catch-up NACK guard at the ack boundary The catch-up-NACK routing guard (fsn <= ackedFsn) had its equality case untested: the common single-frame catch-up maps its only catch-up frame to fsn == ackedFsn exactly, yet the existing test only exercised the strictly-below case (fsn = -2 < ackedFsn = -1). A "<=" -> "<" mutation therefore survived the suite while routing the single (or last) catch-up frame's NACK onto the poison-strike path and laundering the detector. Add testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike, which drives fsn == ackedFsn == -1 and asserts poisonStrikes stays 0. poisonStrikes, not poisonFsn, is the discriminator here: a wrongly-charged strike sets poisonFsn to the rejected fsn -1, which is the "no suspect" sentinel, so a poisonFsn check is blind at this boundary. The mutation charges the strike (0 -> 1) and the test fails; the production guard is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ursorWebSocketSendLoopPoisonFrameTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index d413d82e..d63887db 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -696,6 +696,48 @@ public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Ex }); } + @Test + public void testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike() throws Exception { + // Boundary companion to testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState: + // the COMMON single-frame catch-up maps its only catch-up frame to + // fsn == ackedFsn EXACTLY (the last catch-up frame lands on replayStart-1 == + // ackedFsn). The pre-send routing guard is "fsn <= ackedFsn", so this equality + // case must also skip the poison strike. Without pinning it a "<=" -> "<" + // mutation would slip the single/last catch-up frame's NACK onto the post-send + // poison-strike path and launder the detector -- exactly the hazard the guard + // closes -- yet the strictly-below test above would still pass. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + // 1 catch-up frame (wire seq 0) + 1 data frame (wire seq 1), nothing + // acked (ackedFsn = -1), so fsnAtZero = replayStart(0) - catchUpFrames(1) + // = -1. A NACK of catch-up wire seq 0 maps to fsn -1 == ackedFsn exactly. + setPostCatchUpDataFrameBaseline(loop, -1L, 2L); + assertEquals("precondition: no strikes yet", + 0L, getLongField(loop, "poisonStrikes")); + + deliverRetriableNack(loop, 0, "disk full"); + + // poisonStrikes -- NOT poisonFsn -- is the discriminator at this exact + // boundary: a wrongly-charged strike calls recordHeadRejectionStrike(-1), + // which sets poisonFsn to the rejected fsn -1 -- the "no suspect" + // sentinel -- so a poisonFsn check cannot see the laundering here. It + // still bumps poisonStrikes 0 -> 1, so the correct pre-send routing (no + // strike) shows as poisonStrikes staying 0. A "<=" -> "<" mutation of the + // fsn-vs-ackedFsn guard charges the strike and fails this. + assertEquals("catch-up NACK at fsn == ackedFsn must not charge a poison strike", + 0L, getLongField(loop, "poisonStrikes")); + assertEquals("... and must leave the poison sentinel intact", + -1L, getLongField(loop, "poisonFsn")); + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testPostSendNotWritableNackNeverEscalatesToPoisonTerminal() throws Exception { // RETRIABLE_OTHER (NOT_WRITABLE) is a node-state verdict, not a frame From 031777e27b4452f8f9c228fe7a5d6e19c3849d4d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 23:00:04 +0100 Subject: [PATCH 48/78] Cover the cap-gap budget reset on success The cap-gap settle budget (catchUpCapGapAttempts) counts consecutive catch-up cap gaps across reconnects and resets to 0 on a successful catch-up, so gaps interspersed with successes -- a rolling-cap cluster -- never accumulate to a spurious terminal. The existing latch test only accrues gaps under one fixed cap with no success interleaved, so a dropped reset would have gone unnoticed. Add testSuccessfulCatchUpResetsCapGapBudget: accrue max-1 cap gaps, let a larger-cap node accept the dictionary, then assert the budget is back to 0 and that max-1 further gaps still latch no terminal. Make the stub client's advertised cap mutable so the test can model the rolling cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 54fd645c..c21b7d82 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -321,6 +321,63 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { }); } + @Test + public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { + // The cap-gap settle budget (catchUpCapGapAttempts) counts CONSECUTIVE cap + // gaps across reconnects; a successful catch-up ends the episode and MUST reset + // it to 0 (sendDictCatchUp's final line). Otherwise cap gaps interspersed with + // successful catch-ups -- a rolling-cap cluster where a larger-cap node comes + // and goes -- would accumulate to a spurious terminal over a long-lived sender. + // testCatchUpCapGapRetriesUntilBudgetThenLatches only accrues gaps under one + // fixed cap with no success interleaved, so it cannot pin the reset. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); // too small for a 200-char symbol + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + // Accrue max-1 consecutive cap gaps (each retriable, no terminal). + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + } + assertEquals("precondition: budget accrued to max-1", + maxAttempts - 1, readInt(loop, "catchUpCapGapAttempts")); + + // A larger-cap node returns: the whole dictionary re-registers with + // no cap gap, so the settle budget must reset to 0. + client.cap = 0; // no cap => the 200-char symbol fits one frame + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + assertEquals("a successful catch-up must reset the cap-gap settle budget", + 0, readInt(loop, "catchUpCapGapAttempts")); + + // Behavioural proof the budget is genuinely fresh: max-1 more cap + // gaps still latch NO terminal (they would if the counter had stayed + // at max-1 -- one more gap would have hit the cap and killed the sender). + client.cap = 160; + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("post-reset cap gap must be retriable (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); // fresh budget => still under max => no terminal + } + } finally { + loop.close(); + } + } + }); + } + @Test public void testMirrorOverflowFailsLoud() throws Exception { // ensureSentDictCapacity must latch a terminal -- not silently overflow the @@ -539,7 +596,9 @@ private static long writeVarint(long addr, long value) { // when throwOnSend is set -- raises a transient wire error to model the fresh // connection dropping mid-catch-up. private static final class CatchUpCapturingClient extends WebSocketClient { - private final int cap; + // Mutable so a test can model a rolling-cap cluster: raise it for a node that + // accepts the dictionary, lower it for a smaller-cap node that cap-gaps. + private int cap; private final boolean throwOnSend; private int framesSent; From 0c0b9dc55b1685e4c7219f4fe9fde5b91cae777d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 23:00:17 +0100 Subject: [PATCH 49/78] Cover file-mode split write-ahead dict persist flushPendingRowsSplit's write-ahead dictionary persist (the appendRawEntries fast path) runs only in the exceptional over-cap split path, and both split tests were memory-mode, so the split-times-persist path was never exercised with a live PersistedSymbolDict -- a recovered slot relies on it to rebuild the delta frames. Add testFileModeSplitPersistsDictBeforePublish: a file-mode two-table batch that exceeds the server cap splits into two frames, and the first frame's persist must record both new symbols in .symbol-dict. Dropping the split persist leaves the file empty and fails the test; the non-split file persist test is unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 524063a1..684f0239 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -332,6 +332,55 @@ public void testOversizedTableSplitStrandsNothing() throws Exception { }); } + @Test + public void testFileModeSplitPersistsDictBeforePublish() throws Exception { + // File-mode store-and-forward + a SPLIT flush: a two-table batch whose combined + // size exceeds the server cap splits into one frame per table + // (flushPendingRowsSplit). The first split frame's write-ahead persist + // (persistNewSymbolsBeforePublish, the appendRawEntries fast path) must record + // the batch's new symbols in .symbol-dict BEFORE the frames publish, so a + // recovered / orphan-drained slot can rebuild what the delta frames reference. + // The other split tests run in memory mode, so this is the only coverage of the + // split x persist path with a live PersistedSymbolDict. + Path sfDir = Files.createTempDirectory("qwp-sf-split-persist"); + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + String pad = new String(new char[60]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig(config)) { + // Two tables, two new symbols, ONE flush -> the combined message + // exceeds cap 150 and splits into two frames. + sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check .symbol-dict while the sender is live: a fully-drained + // close would unlink it. The split's first-frame write-ahead + // persist must have recorded BOTH new symbols. + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("split-flush persist must record alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("split-flush persist must record bravo", containsUtf8(dict, "bravo")); + } + + Assert.assertEquals("the oversized two-table batch must split into 2 frames", + 2, handler.batches.size()); + } + }); + } finally { + rmDir(sfDir); + } + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; From bbc01a82da19ca9097b8a81c6e1617fc22729bc8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 02:00:11 +0100 Subject: [PATCH 50/78] Cover split-preflight and torn-dict guard gaps Address review findings on the delta symbol-dictionary change. Tests (close two coverage gaps): - SelfSufficientFramesTest pins the split-flush pre-flight baseline advance: a large-delta two-table split whose second frame fits only with an empty delta. Removing the simBaseline advance mis-sizes that frame with the delta and wrongly rejects a shippable batch. - CursorWebSocketSendLoopOrphanTailTest pins that a zero-count delta frame anchors recoveredMaxSymbolId at its baseline, not 0. Skipping it would under-strand a torn dictionary and risk a silent id shift. Docs (correct load-bearing recovery semantics): - seedGlobalDictionaryFromPersisted no longer claims the drainer always self-heals: it does so only when a surviving frame straddles the tear; a higher-baseline survivor quarantines the slot, a deliberate conservative over-strand. - maxSymbolDeltaEnd documents that a zero-count delta frame contributes its deltaStart, not 0. - The catch-up cap-gap budget documents that a transient reconnect never increments or resets the counter (only a successful catch-up resets it), so a transient cannot burn the terminal budget. Hardening: - writeVarint and putVarint assert a non-negative value: the signed loop writes one truncated byte for a negative long while varintSize returns 10. - PersistedSymbolDict.open diverts a file at exactly Integer.MAX_VALUE to a clean re-create instead of a doomed 2GB malloc (> becomes >=). - readLoadedSymbols bounds its varint decode (shift > 35) like its sibling parsers. Production behavior changes are limited to the two varint asserts, the >= boundary, and the defensive varint bound; the rest is tests and documentation. Full delta-dict suite green on JDK 25 with -ea. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/NativeBufferWriter.java | 7 ++ .../qwp/client/QwpWebSocketSender.java | 20 +++++- .../sf/cursor/CursorWebSocketSendLoop.java | 46 ++++++++----- .../qwp/client/sf/cursor/MmapSegment.java | 13 +++- .../client/sf/cursor/PersistedSymbolDict.java | 31 ++++++--- .../qwp/client/SelfSufficientFramesTest.java | 69 +++++++++++++++++++ ...CursorWebSocketSendLoopOrphanTailTest.java | 47 +++++++++++++ 7 files changed, 200 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index cb6ebb92..86343772 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -81,8 +81,14 @@ public static int varintSize(long value) { * {@code addr} and returns the address just past the last byte. The canonical * raw-address varint writer shared by the SF cursor's persisted dictionary and * catch-up frame builder. + *

+ * {@code value} must be non-negative: the signed {@code value > 0x7F} loop emits + * a SINGLE truncated byte for a negative long, whereas {@link #varintSize} + * returns 10 for it -- a size/write mismatch that would corrupt the stream. All + * callers pass ids/lengths/counts (non-negative); the assert pins that contract. */ public static long writeVarint(long addr, long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; while (value > 0x7F) { Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); value >>>= 7; @@ -320,6 +326,7 @@ public void putUtf8(CharSequence value) { */ @Override public void putVarint(long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; ensureCapacity(10); // max varint bytes long addr = bufferPtr + position; while (value > 0x7F) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4f7c374b..70a7e877 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3876,9 +3876,23 @@ private void resetSymbolDictStateForNewConnection() { * mirror, leaving only this producer diverged. Detect it here -- the surviving * frames reference an id at or beyond the recovered dictionary size -- and fail * clean: the affected data must be resent, matching the design's torn-dict - * "resend required" contract. The recovered data is not lost; the background - * drainer still drains this slot (its mirror self-heals from the frames). Only a - * host crash reaches this -- a process crash keeps the page cache, so the + * "resend required" contract. + *

+ * The background drainer self-heals the mirror ONLY when a surviving frame + * STRADDLES the tear ({@code deltaStart <= pd.size() < deltaStart + deltaCount}): + * such a frame carries the torn-off ids in its own delta and + * {@code accumulateSentDict} re-registers them, so the drainer drains the slot. + * But when the symbol-introducing frames were already acked and trimmed and only + * a HIGHER-baseline frame survives ({@code deltaStart > pd.size()} -- e.g. a + * commit or a symbol-reusing frame, since {@code beginMessage} always sets the + * delta flag), the drainer's own replay guard ({@code deltaStart > sentDictCount}) + * fires too and quarantines the slot: the recorded bytes are not silently lost, + * but the slot is NOT auto-drained -- it must be resent. That is a deliberate + * CONSERVATIVE over-strand -- the guard keys on {@code deltaStart}, not on the + * frame's actual highest referenced id, to avoid parsing row data at recovery, + * so it may reject a frame whose rows reference only ids the truncated + * dictionary still holds. It fails clean rather than risk a silent id shift. + * Only a host crash reaches this -- a process crash keeps the page cache, so the * write-ahead ordering keeps the dictionary a superset of the frames. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index f55f47c8..1ea87523 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -133,17 +133,30 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); - // Settle budget for the symbol-dict catch-up cap gap: how many CONSECUTIVE - // reconnect attempts may find a single dictionary entry too large for the fresh - // server's advertised batch cap before the sender latches a terminal. A - // homogeneous cluster never trips it -- an entry that fit its data frame under a - // cap always fits its bare catch-up frame under that same cap -- so this only - // affects a heterogeneous / rolling-cap cluster, where a failover to a - // smaller-cap node can hit it for an entry an earlier node accepted. Retrying - // rides out the transient window until a larger-cap node returns; only a - // persistent gap (every reachable node too small for this many attempts) latches - // terminal, matching the orphan drainer's DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS. - // A successful catch-up resets the counter (see sendDictCatchUp). + // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts + // -- catch-ups that reached a fresh server and found a single dictionary entry + // too large for its advertised batch cap -- may occur, with no intervening + // SUCCESSFUL catch-up, before the sender latches a terminal. This is a SANCTIONED + // terminal (a genuine cluster batch-size capability gap), the connect-time analog + // of the orphan drainer's durable-ack capability gap + // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips + // it -- an entry that fit its data frame under a cap always fits its bare catch-up + // frame under that same cap -- so it only affects a heterogeneous / rolling-cap + // cluster, where a failover to a smaller-cap node can hit it for an entry an + // earlier node accepted. Retrying rides out the transient window until a + // larger-cap node returns; only a persistent gap (this many cap gaps with no + // successful catch-up in between) latches. + // + // Budget accounting (satisfies "a transient must never burn the terminal + // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a + // node is reached and an entry is oversized, and resets ONLY when a catch-up fully + // succeeds. A TRANSIENT reconnect (connect refuse, upgrade/role failure) never + // reaches the catch-up, so it NEITHER increments nor resets the counter -- it only + // lengthens the wall-clock settle window. The terminal therefore always requires + // this many GENUINE cap gaps; a transient can never inflate it. Deliberately NOT + // reset on a mere successful RECONNECT: a reconnect to the small-cap node itself + // produces the cap gap, so resetting there would stop a persistent gap from ever + // latching -- the reset must gate on a successful CATCH-UP, not on connecting. private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even @@ -218,11 +231,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // for the connection's lifetime (a reconnect may need the whole dictionary at // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; - // Consecutive reconnect attempts whose symbol-dict catch-up found an entry too - // large for the fresh server's batch cap (see MAX_CATCHUP_CAP_GAP_ATTEMPTS). A - // successful catch-up resets it; it is NOT reset per connection -- it measures - // the cap-gap episode across reconnects so a persistent gap eventually latches. - // I/O-thread-only. + // Cap-gap attempts -- catch-ups that reached a node and found an entry too large + // for its batch cap -- since the last SUCCESSFUL catch-up (see + // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful + // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments + // nor resets it. NOT reset per connection -- it measures the cap-gap episode + // across reconnects so a persistent gap eventually latches. I/O-thread-only. private int catchUpCapGapAttempts; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index a7ee42ec..50fc18ff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -523,9 +523,16 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in } /** - * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) that - * any symbol-dict delta frame in this segment references, or {@code 0} when no - * such frame carries a symbol. Read-only walk over the recovered frames, used + * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any + * delta-flagged frame in this segment carries, or {@code 0} only when NO + * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit + * or a symbol-reusing frame -- the encoder always sets the delta flag) still + * contributes its {@code deltaStart}, i.e. the producer's baseline at encode + * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that + * baseline so a dictionary torn below it is detected -- a conservative + * over-strand (it may over-reject a frame whose rows reference only surviving + * ids), never an under-strand that could silently shift the dense id map. + * Read-only walk over the recovered frames, used * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, * out-of-order page loss) below the ids the surviving frames still reference: if * the max here reaches at or beyond the recovered dictionary size, a resuming diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 70cadb19..51c1b19b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -170,17 +170,19 @@ public static PersistedSymbolDict open(String slotDir) { public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; - // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be - // reopened: openExisting reads it into ONE int-sized native buffer, and - // the (int) cast of a >2GB length is either negative (malloc rejects it), - // exactly zero (getInt then reads 4 bytes past a zero-size allocation), or - // a small positive prefix (whose validLen < len branch would then - // DESTRUCTIVELY truncate the multi-GB file). Recreate empty instead -- - // fail-clean, exactly like every other unreadable-file case here, so the - // sender falls back to full self-sufficient frames. Reaching this needs - // ~100M+ distinct symbols on one slot (far past realistic symbol - // cardinality); the guard keeps the read/write size boundary safe anyway. - if (existing > Integer.MAX_VALUE) { + // A dictionary that grew to or past Integer.MAX_VALUE cannot be reopened: + // openExisting reads it into ONE int-sized native buffer. PAST 2GiB the + // (int) cast is either negative (malloc rejects it), exactly zero (getInt + // then reads 4 bytes past a zero-size allocation), or a small positive prefix + // (whose validLen < len branch would then DESTRUCTIVELY truncate the multi-GB + // file); AT exactly Integer.MAX_VALUE the cast is exact but the ~2GB malloc is + // doomed to OutOfMemoryError. The >= guard short-circuits every case to a + // clean re-create instead of the doomed allocation -- fail-clean, exactly like + // every other unreadable-file case here, so the sender falls back to full + // self-sufficient frames. Reaching this needs ~100M+ distinct symbols on one + // slot (far past realistic symbol cardinality); the guard keeps the read/write + // size boundary safe anyway. + if (existing >= Integer.MAX_VALUE) { LOG.warn("symbol dict {} too large ({} bytes) to reopen; recreating empty", filePath, existing); return openFresh(ff, filePath); } @@ -463,6 +465,13 @@ public ObjList readLoadedSymbols() { break; } shift += 7; + if (shift > 35) { + // Bound the varint like decodeVarint / appendRawEntries / + // readVarintAt: a canonical length is <= 5 bytes. open() already + // CRC-validated these bytes, so this is defensive only; the + // p + len > limit check below then rejects the over-long run. + break; + } } if (p + len > limit) { break; // defensive: torn tail (should not happen past parse in open) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 684f0239..5729aa44 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -381,6 +381,75 @@ public void testFileModeSplitPersistsDictBeforePublish() throws Exception { } } + @Test + public void testSplitPreflightAdvancesBaselineSoLaterFramesArentSizedWithTheDelta() throws Exception { + // Regression for the split pre-flight baseline advance in flushPendingRowsSplit + // (the "Mirror advanceSentMaxSymbolId" step). Only the FIRST split frame ships + // the batch's symbol-dict delta; the rest ship an EMPTY delta and reference ids + // the first frame already registered. The pre-flight size pass must advance + // simBaseline after the first table so it STOPS adding combinedDeltaEntriesLen + // to the later frames' estimated sizes. Without that advance, a later table + // whose real (empty-delta) frame fits the cap is mis-estimated as still carrying + // the whole delta and wrongly rejected with "single table batch too large" -- + // discarding a legitimately shippable batch (fail-closed data loss). + // + // Shape (memory mode, delta enabled): a LARGE combined delta (two ~64-char + // symbols) rides only the first split frame. The first table (added first, so + // the first split frame) has a tiny body, so delta + body1 fits the cap. The + // second table has a big body: body2 alone fits the cap, but delta + body2 does + // NOT. The real code splits and ships both frames; the un-advanced pre-flight + // would size the second frame WITH the delta and throw. Table order is + // insertion order (CharSequenceObjHashMap.keys()), so t1 is the delta frame. + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(200); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Two long symbols => a large combined delta section (~130 bytes) that + // rides ONLY the first split frame. The symbol STRINGS live in the delta, + // not in either table body (the body carries only the varint global id). + String longSymA = new String(new char[64]).replace('\0', 'a'); + String longSymB = new String(new char[64]).replace('\0', 'b'); + String bigPad = new String(new char[100]).replace('\0', 'x'); + // auto_flush off so both rows batch into one flush (byte-based auto-flush + // is otherwise clamped under the cap and would flush before the split). + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) { + // t1 (added first -> first split frame): carries the whole delta but a + // tiny body, so delta + body1 fits the 200-byte cap. + sender.table("t1").symbol("s", longSymA).longColumn("v", 1L).atNow(); + // t2 (second split frame): empty delta but a big body. body2 alone + // fits the cap; delta + body2 does NOT -- the mis-size the advance + // prevents. + sender.table("t2").symbol("s", longSymB).stringColumn("p", bigPad).longColumn("v", 2L).atNow(); + // Must NOT throw: with the baseline advanced, t2's frame is sized + // WITHOUT the delta and fits. A broken advance throws "too large" here. + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("the batch must split into 2 frames, neither spuriously rejected", + 2, handler.batches.size()); + byte[] f1 = handler.batches.get(0); + byte[] f2 = handler.batches.get(1); + // First split frame ships the whole delta (both new symbols, ids 0 and 1). + Assert.assertEquals("first split frame deltaStart must be 0", + 0, readVarint(f1, DELTA_START_OFFSET)); + Assert.assertEquals("first split frame ships both new symbols", + 2, readVarint(f1, DELTA_START_OFFSET + 1)); + // Second split frame carries an EMPTY delta above the advanced baseline -- + // the whole point: it is not re-sized (or re-sent) with the delta. + Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)", + 2, readVarint(f2, DELTA_START_OFFSET)); + Assert.assertEquals("second split frame carries no new symbols", + 0, readVarint(f2, DELTA_START_OFFSET + 1)); + } + }); + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index ed3989ee..bf16f3b0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -253,6 +253,53 @@ public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception }); } + @Test + public void testZeroCountDeltaFrameAnchorsRecoveredMaxSymbolIdAtItsBaseline() throws Exception { + // A committed delta frame that introduces NO new symbol (deltaCount == 0 -- a + // commit frame, or one whose rows only re-use existing ids) still carries + // deltaStart == the producer's baseline at encode time, because beginMessage + // ALWAYS sets FLAG_DELTA_SYMBOL_DICT. maxSymbolDeltaEnd counts it as + // deltaStart + deltaCount == deltaStart (NOT 0), so recoveredMaxSymbolId == + // deltaStart - 1 even though the frame introduces nothing. + // + // This is the mechanism behind the torn-dict guard's deliberate CONSERVATIVE + // over-strand (see seedGlobalDictionaryFromPersisted): if a host crash tears + // the persisted dictionary below such a frame's baseline while its + // symbol-introducing predecessors were already acked and trimmed, both the + // seed-time guard (recoveredMaxSymbolId >= pd.size()) and the drainer's replay + // guard (deltaStart > sentDictCount) fire and quarantine the slot -- fail-clean + // "resend required" -- even though the frame's rows may reference only ids the + // truncated dictionary still holds. + // + // Counting the zero-count frame's baseline is load-bearing SAFETY: a "fix" that + // skipped zero-count frames (returning 0 for them) would UNDER-strand and let a + // genuinely torn dictionary through, silently shifting the dense id map. This + // pins that a zero-count delta frame is anchored at its baseline, not skipped. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + // fsn 0: commit-bearing delta frame that genuinely registers ids 0..4. + appendDeltaFrame(engine, false, 0, 5); + // fsn 1: commit-bearing delta frame with deltaStart 10, deltaCount 0 -- + // introduces NOTHING, but its baseline (10) sits ABOVE every id any + // surviving frame actually introduces (max 4). Models a commit / + // symbol-reusing frame emitted after ids 5..9 were registered by + // predecessor frames that have since been acked and trimmed away. + appendDeltaFrame(engine, false, 10, 0); + } + try (CursorSendEngine engine = newEngine()) { + assertTrue(engine.wasRecoveredFromDisk()); + assertEquals("both frames are commit-bearing", 1L, engine.recoveredCommitBoundaryFsn()); + // The zero-count frame drives recoveredMaxSymbolId to 9 (its baseline + // 10, minus 1), NOT to 4 (the highest id any surviving frame actually + // introduces) and NOT to 0 (which skipping it would yield). This + // inflation is exactly what makes seedGlobalDictionaryFromPersisted + // over-reject a dictionary holding ids 0..4 (size 5). + assertEquals("a zero-count delta frame anchors recoveredMaxSymbolId at its baseline-1", + 9L, engine.recoveredMaxSymbolId()); + } + }); + } + // --------------------------------------------------------------------- // harness // --------------------------------------------------------------------- From 39e9e52fd3ce0323cc5fafac87d3280a0a0dde47 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 12:22:39 +0100 Subject: [PATCH 51/78] Cover the persisted symbol-dict short-write path The failed-publish regression tests exercise a persist that SUCCEEDED (the publish is what fails), so nothing pinned the persist-FAILURE trigger: a short write (disk full / quota) mid-persist. Add a ShortWriteOnceFacade that lands one entry append short and two tests -- one per production persist path (appendRawEntries fast path, appendSymbols slow path) -- asserting the load-bearing idempotency invariant the write-ahead persist relies on: a short write throws WITHOUT advancing size()/appendOffset, so a retry keyed off pd.size() re-persists the same range at the same offset and recovers a gap-free, duplicate-free dictionary. Verified as regression guards: reordering the production code to advance size before the written==len check fails both tests with "short write must NOT advance size expected:<0> but was:<2>". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/PersistedSymbolDictTest.java | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index eb79e016..677e12e2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -146,6 +146,78 @@ public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { }); } + @Test + public void testAppendRawEntriesShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception { + // The producer's FAST path persists a frame's already-encoded delta bytes + // via appendRawEntries. A short write (disk full / quota) mid-persist must + // throw WITHOUT advancing size()/appendOffset, so a retry keyed off size() + // re-writes the identical bytes at the same offset and recovers a gap-free, + // duplicate-free dictionary. The failed-PUBLISH regressions + // (DeltaDictRecoveryTest) exercise a persist that SUCCEEDED; only this pins + // the persist-FAILURE trigger, whose idempotency the write-ahead + // (persistNewSymbolsBeforePublish, resuming from pd.size()) relies on. + assertMemoryLeak(() -> { + Path src = Files.createTempDirectory("qwp-symdict-src"); + Path dst = Files.createTempDirectory("qwp-symdict-dst"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + + // Encode the range once to obtain its on-disk [len][utf8]... bytes. + PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + encoded.appendSymbols(dict, 0, 1); + encoded.close(); + + PersistedSymbolDict source = PersistedSymbolDict.open(src.toString()); + try { + long addr = source.loadedEntriesAddr(); + int rawLen = source.loadedEntriesLen(); + int count = source.size(); + + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + PersistedSymbolDict d = PersistedSymbolDict.open(ff, dst.toString()); + Assert.assertNotNull(d); + try { + ff.armed = true; // the next entry append lands short + try { + d.appendRawEntries(addr, rawLen, count); + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); + } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); + + // Retry the SAME bytes (the facade auto-disarmed): the write + // lands at the unchanged offset, overwriting the torn prefix. + d.appendRawEntries(addr, rawLen, count); + Assert.assertEquals(2, d.size()); + } finally { + d.close(); + } + } finally { + source.close(); + } + + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + PersistedSymbolDict reopened = PersistedSymbolDict.open(dst.toString()); + try { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("MSFT", symbols.getQuick(1)); + } finally { + reopened.close(); + } + } finally { + rmDir(src); + rmDir(dst); + } + }); + } + @Test public void testAppendSymbolsBatchWritesDenseRange() throws Exception { // appendSymbols persists a whole id range in one write (the hot-path @@ -204,6 +276,60 @@ public void testAppendSymbolsBatchWritesDenseRange() throws Exception { }); } + @Test + public void testAppendSymbolsShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception { + // The producer's SLOW path (re-encode the [pd.size()..batchMax] suffix after + // a prior partial persist) writes via appendSymbols. A short write (disk full + // / quota) must throw WITHOUT advancing size()/appendOffset, so a retry keyed + // off size() re-persists the SAME range at the SAME offset and recovers a + // gap-free, duplicate-free dictionary. A regression that advanced size before + // the written==len check would strand a torn/duplicated dictionary here. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("GOOG"); // id 1 + + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + PersistedSymbolDict d = PersistedSymbolDict.open(ff, dir.toString()); + Assert.assertNotNull(d); + try { + ff.armed = true; // the next entry append lands short + try { + d.appendSymbols(dict, 0, 1); + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); + } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); + + // Retry the SAME range (the facade auto-disarmed): re-writes at + // the unchanged offset, overwriting the torn bytes. + d.appendSymbols(dict, 0, 1); + Assert.assertEquals(2, d.size()); + } finally { + d.close(); + } + + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); + } finally { + reopened.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testBadMagicIsRecreatedEmpty() throws Exception { assertMemoryLeak(() -> { @@ -776,4 +902,24 @@ public long length(String path) { return (long) Integer.MAX_VALUE + 1L; } } + + /** + * Lands ONE armed entry append short -- writes {@code len-1} of the {@code len} + * requested bytes and reports {@code len-1} -- reproducing a disk-full / quota + * short write mid-persist. Fires only on an entry append (offset past the + * 8-byte header), never the header write, and disarms after firing so the retry + * writes cleanly. + */ + private static final class ShortWriteOnceFacade extends DelegatingFilesFacade { + boolean armed; + + @Override + public long write(int fd, long addr, long len, long offset) { + if (armed && offset > 0 && len > 1) { + armed = false; + return INSTANCE.write(fd, addr, len - 1, offset); + } + return INSTANCE.write(fd, addr, len, offset); + } + } } From 265b703468890f402f8b3d3cd46816d55b5e4bc0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 12:45:08 +0100 Subject: [PATCH 52/78] Harden the symbol-dict fd close and split sizing Three review follow-ups on the delta symbol-dictionary change: - PersistedSymbolDict relinquishes fd (sets it to -1) before each in-branch ff.close and guards the catch-block closes with fd >= 0, so a throwing in-branch close cannot double-close via the catch. Completes the null-before-free discipline the buffers already use. - QwpWebSocketEncoderTest pins the split path's context-free-body invariant: a table's body bytes must encode identically solo and combined, since flushPendingRowsSplit sizes each frame from the combined encode's splitFrameBodyBytes rather than re-measuring. A future column encoder that carried cross-table state now fails here instead of silently stranding a deferred prefix. - CursorSendEngine imports QwpConstants instead of spelling it out fully-qualified eight times, matching its sibling send loop. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 17 ++--- .../client/sf/cursor/PersistedSymbolDict.java | 20 ++++-- .../qwp/client/QwpWebSocketEncoderTest.java | 68 +++++++++++++++++++ 3 files changed, 92 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index dde24142..2c9746af 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; import io.questdb.client.std.ObjList; @@ -307,10 +308,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DEFER_COMMIT, + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_SIZE ); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; @@ -332,10 +333,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // returns 0 when no such frame carries a symbol, yielding -1 here. // Computed before the I/O loop or producer append; single-threaded. this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE, + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT, + QwpConstants.HEADER_SIZE, recoveredCommitBoundaryFsn) - 1L; } else { // Fresh start with no recovered segments. Any stale diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 51c1b19b..83371201 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -532,7 +532,9 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // null after free so the catch below cannot double-free if ff.close throws - ff.close(fd); + int fdToClose = fd; + fd = -1; // relinquish before close so the catch cannot double-close if close throws + ff.close(fdToClose); return null; } // Parse complete, CRC-valid entries after the header; stop at the first @@ -622,7 +624,9 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, entriesAddr = 0L; entriesLen = 0; } - ff.close(fd); + int fdToClose = fd; + fd = -1; // relinquish before close so the catch cannot double-close if close throws + ff.close(fdToClose); return null; } return new PersistedSymbolDict(ff, fd, validLen, count, entriesAddr, entriesLen); @@ -638,7 +642,9 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } - ff.close(fd); + if (fd >= 0) { // a branch that already closed fd relinquished it to -1 + ff.close(fd); + } LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); return null; } @@ -660,7 +666,9 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0); long written = ff.write(fd, hdr, HEADER_SIZE, 0); if (written != HEADER_SIZE) { - ff.close(fd); + int fdToClose = fd; + fd = -1; // relinquish before close so the catch cannot double-close if close throws + ff.close(fdToClose); ff.remove(filePath); LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; @@ -670,7 +678,9 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte // malloc cannot realistically OOM), but close the fd against a future // edit so it cannot leak -- mirroring openExisting's error handling. - ff.close(fd); + if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing + ff.close(fd); + } LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); return null; } finally { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 4d7d7931..b21d2e25 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -1284,6 +1284,74 @@ public void testReset() throws Exception { }); } + @Test + public void testTableBodyEncodingIsContextFree() throws Exception { + // The split-flush path (QwpWebSocketSender.flushPendingRowsSplit) sizes each + // per-table frame ARITHMETICALLY from splitFrameBodyBytes -- the body byte + // count captured during the COMBINED encode in flushPendingRows -- instead of + // re-encoding to measure. That is sound only while a table's body bytes are + // context-free: identical whether the table is encoded solo or as the k-th + // table after other tables and the delta section. Today every column encoder + // is stateless and symbol cells carry absolute global ids, so the property + // holds; this pins it so a future column encoder that ever carried + // cross-table state (which would break the arithmetic sizing and could strand + // a deferred prefix mid-split) fails loudly here rather than silently in + // production. + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer t0 = new QwpTableBuffer("alpha"); + QwpTableBuffer t1 = new QwpTableBuffer("bravo"); + QwpTableBuffer t2 = new QwpTableBuffer("charlie")) { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + int aapl = dict.getOrAddSymbol("AAPL"); // 0 + int goog = dict.getOrAddSymbol("GOOG"); // 1 + int msft = dict.getOrAddSymbol("MSFT"); // 2 + + // Distinct schemas + a symbol column (absolute global ids) so a + // positional / cross-table encoder bug would shift the body bytes. + t0.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("AAPL", aapl); + t0.getOrCreateColumn("v", TYPE_LONG, false).addLong(1L); + t0.nextRow(); + + t1.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("GOOG", goog); + t1.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(2.5); + t1.getOrCreateColumn("s", TYPE_VARCHAR, true).addString("hello"); + t1.nextRow(); + + t2.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("MSFT", msft); + t2.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP).addLong(1_000_000L); + t2.nextRow(); + + QwpTableBuffer[] tables = {t0, t1, t2}; + int confirmedMaxId = -1; + int batchMaxId = 2; + + // Combined encode: capture each table's body bytes exactly as + // flushPendingRows' splitFrameBodyBytes does (position delta per addTable). + int[] combinedBody = new int[tables.length]; + encoder.beginMessage(tables.length, dict, confirmedMaxId, batchMaxId); + int bodyStart = encoder.getBuffer().getPosition(); + for (int i = 0; i < tables.length; i++) { + encoder.addTable(tables[i]); + int bodyEnd = encoder.getBuffer().getPosition(); + combinedBody[i] = bodyEnd - bodyStart; + bodyStart = bodyEnd; + } + + // Solo encode each table under the SAME baseline/batch max, as the + // split publish loop does, and assert the body bytes match the capture. + for (int i = 0; i < tables.length; i++) { + encoder.beginMessage(1, dict, confirmedMaxId, batchMaxId); + int soloBodyStart = encoder.getBuffer().getPosition(); + encoder.addTable(tables[i]); + int soloBody = encoder.getBuffer().getPosition() - soloBodyStart; + Assert.assertEquals("table " + i + " body must encode identically solo vs combined " + + "(splitFrameBodyBytes relies on a context-free body)", combinedBody[i], soloBody); + } + } + }); + } + @Test public void testVersionByteInHeader() throws Exception { assertMemoryLeak(() -> { From 2fa5cd9bb24b219f8f83895545fce77de6441b57 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:24:38 +0100 Subject: [PATCH 53/78] Recover full-dict SF slots without bricking build A store-and-forward slot written in full-dict fallback -- the .symbol-dict could not open when writing, so every frame re-ships the whole dictionary from id 0 -- leaves self-sufficient frames behind but no side-file. On recovery the engine opens a fresh empty .symbol-dict, so the surviving frames out-reach it, and the sender's seed-time guard mistook the empty dictionary for a host-crash tear: it failed Sender.build() for a slot the orphan drainer drains fine. CursorSendEngine now tells the two cases apart with a new maxSymbolDeltaStart walk (MmapSegment, SegmentRing). When the persisted dictionary is a subset of the ids the frames reference but every such frame is self-sufficient (maxSymbolDeltaStart == 0), the engine discards the empty side-file so isDeltaDictEnabled() reports false and the slot recovers in full-dict mode, exactly how it was written. A genuine torn delta dictionary keeps a deltaStart > 0 frame and still fails clean. testFullDictFramesRecoverInFullDictModeInsteadOfBricking covers the new path -- red without the discard, green with it. The torn-delta fail-clean tests are unchanged. Also regroups CursorWebSocketSendLoop's six new mutable delta-dict fields with the other mutable fields (deltaDictEnabled stays final, in its alphabetical slot) and adds the missing thousands separator to a test's 16_384 batch-size literal. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 25 +++-- .../client/sf/cursor/CursorSendEngine.java | 30 ++++++ .../sf/cursor/CursorWebSocketSendLoop.java | 93 ++++++++++--------- .../qwp/client/sf/cursor/MmapSegment.java | 56 +++++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 19 ++++ .../qwp/client/DeltaDictRecoveryTest.java | 81 ++++++++++++++++ ...WebSocketSendLoopCatchUpAlignmentTest.java | 2 +- 7 files changed, 252 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 70a7e877..c9f7203c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3900,14 +3900,23 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { return; } // Run the torn-dictionary guard BEFORE the empty-dictionary short-circuit - // below: a TOTAL tear (pd.size() == 0) with surviving symbol-bearing frames - // must fail clean too, not slip through. Such a frame starts at deltaStart=0 - // and self-heals the I/O-thread catch-up mirror, so the send loop's replay - // guard (deltaStart > sentDictCount) never fires -- this seed-time guard is - // then the only defense against the producer resuming unseeded and silently - // reusing ids the frames already define. A genuinely empty slot (no - // symbol-bearing frames) has recoveredMaxSymbolId() == -1, so -1 >= 0 is false - // and the pd.size() == 0 return below still fires. + // below: a TOTAL tear (pd.size() == 0) of a DELTA dictionary with surviving + // symbol-bearing frames must fail clean too, not slip through. Such a frame + // starts at deltaStart=0 and self-heals the I/O-thread catch-up mirror, so the + // send loop's replay guard (deltaStart > sentDictCount) never fires -- this + // seed-time guard is then the only defense against the producer resuming + // unseeded and silently reusing ids the frames already define. A genuinely + // empty slot (no symbol-bearing frames) has recoveredMaxSymbolId() == -1, so + // -1 >= 0 is false and the pd.size() == 0 return below still fires. + // + // This guard fires ONLY for a torn DELTA dictionary. The other way the frames + // can out-reach the recovered dictionary -- a slot written in FULL-DICT + // fallback (the dictionary never opened when writing, every frame + // self-sufficient at deltaStart=0), then recovered against a fresh empty one -- + // is caught upstream in CursorSendEngine, which discards the empty side-file so + // isDeltaDictEnabled() is false and this seed never runs. Those frames need no + // dictionary; failing clean here would needlessly brick build() for a slot the + // orphan drainer drains fine. if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { throw new LineSenderException( "recovered store-and-forward symbol dictionary is a subset of the surviving frames " diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 2c9746af..88a59735 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -338,6 +338,36 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man QwpConstants.FLAG_DELTA_SYMBOL_DICT, QwpConstants.HEADER_SIZE, recoveredCommitBoundaryFsn) - 1L; + // Full-dict-fallback recovery. When the persisted .symbol-dict is a + // SUBSET of the ids the surviving frames reference + // (recoveredMaxSymbolId >= its size) YET every such frame is + // self-sufficient (maxSymbolDeltaStart == 0 -- a full-dict frame that + // re-registers its dictionary from id 0), the slot was written in + // full-dict fallback: the dictionary never opened when writing, so no + // side-file exists and this recovery opened a FRESH EMPTY one. Those + // frames replay with no dictionary, so discard the empty side-file and + // recover in full-dict mode -- isDeltaDictEnabled() then reports false + // and the producer + send loop both run full-dict, exactly as the slot + // was written. Without this the sender's seed-time guard would treat the + // empty dictionary as a host-crash tear and brick build(), even though + // the orphan drainer drains the same frames fine. A genuine torn DELTA + // dictionary keeps a frame with deltaStart > 0 (maxSymbolDeltaStart > 0) + // and is NOT discarded here: it still fails clean at seed time, since + // the ids its delta frames reference cannot be rebuilt without the lost + // dictionary. The recoveredMaxSymbolId >= size guard means this never + // fires for a slot whose dictionary is intact, nor for an empty slot + // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recovered.maxSymbolDeltaStart( + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT, + QwpConstants.HEADER_SIZE, + recoveredCommitBoundaryFsn) == 0L) { + persistedDictInProgress.close(); + persistedDictInProgress = null; + } } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 1ea87523..0c17614f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -168,6 +168,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // True when this loop delta-encodes symbol dictionaries: it keeps a reconnect + // catch-up mirror (sentDict*) and re-registers the whole dictionary on a fresh + // server before replaying delta frames. See the sentDict* fields in the mutable + // section for the full mechanism. Gates all the delta-dict state. + private final boolean deltaDictEnabled; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -211,51 +216,6 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); - // Delta symbol dictionary catch-up state (see swapClient). Active in memory - // mode, and in disk mode whenever the per-slot persisted dictionary opened -- - // fresh slots included, not just recovered / orphan-drained ones. On a - // recovered / orphan-drained slot the constructor additionally SEEDS sentDict* - // from that persisted dictionary; a fresh slot starts with an empty mirror and - // grows it as frames are sent. - // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every - // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in - // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that - // on reconnect it can re-register the whole dictionary on the fresh server - // (which discards its dictionary on every disconnect) before replaying frames - // whose deltas start above id 0. All of this is touched only by the I/O thread. - // Footprint note: this mirror is a SECOND copy of the dictionary -- the same - // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- - // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a - // memory-mode connection's steady-state dictionary footprint is ~2x the symbol - // set. It is bounded by distinct-symbol count (not per-row) and never trimmed - // for the connection's lifetime (a reconnect may need the whole dictionary at - // any moment), so it cannot be dropped; it is an intentional cost of the feature. - private final boolean deltaDictEnabled; - // Cap-gap attempts -- catch-ups that reached a node and found an entry too large - // for its batch cap -- since the last SUCCESSFUL catch-up (see - // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful - // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments - // nor resets it. NOT reset per connection -- it measures the cap-gap episode - // across reconnects so a persistent gap eventually latches. I/O-thread-only. - private int catchUpCapGapAttempts; - // True once a real ring frame (data or commit) has been sent on the CURRENT - // connection, as opposed to only the dictionary catch-up. The catch-up - // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies - // "the head frame was sent": onClose's poison-strike gate and - // handleServerRejection's pre-send gate key off THIS instead. Without it, a - // transient outage AFTER the catch-up but BEFORE the first data frame (a - // flapping LB/middlebox that accepts the upgrade + catch-up then closes) would - // be mistaken for a deterministic head-frame rejection and escalate to a - // PROTOCOL_VIOLATION terminal -- breaking the store-and-forward "retry a - // transient outage forever" contract. Reset per connection in - // setWireBaselineWithCatchUp; set in trySendOne after a successful send. - private boolean dataFrameSentThisConnection; - private long sentDictBytesAddr; - private int sentDictBytesCapacity; - private int sentDictBytesLen; - private int sentDictCount; - // End position (native address) written by the last readVarintAt() call. - private long varintEnd; private final CountDownLatch shutdownLatch = new CountDownLatch(1); private final AtomicLong totalAcks = new AtomicLong(); // Counters for observability of the durable-ack path. Both are zero @@ -279,6 +239,49 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); + // Delta symbol dictionary catch-up state (see swapClient, deltaDictEnabled). + // Active in memory mode, and in disk mode whenever the per-slot persisted + // dictionary opened -- fresh slots included, not just recovered / orphan-drained + // ones. On a recovered / orphan-drained slot the constructor additionally SEEDS + // sentDict* from that persisted dictionary; a fresh slot starts with an empty + // mirror and grows it as frames are sent. The loop mirrors, in sentDict*, every + // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in + // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that on + // reconnect it can re-register the whole dictionary on the fresh server (which + // discards its dictionary on every disconnect) before replaying frames whose + // deltas start above id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed for + // the connection's lifetime (a reconnect may need the whole dictionary at any + // moment), so it cannot be dropped; it is an intentional cost of the feature. + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + private int sentDictCount; + // Cap-gap attempts -- catch-ups that reached a node and found an entry too large + // for its batch cap -- since the last SUCCESSFUL catch-up (see + // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful + // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments + // nor resets it. NOT reset per connection -- it measures the cap-gap episode + // across reconnects so a persistent gap eventually latches. I/O-thread-only. + private int catchUpCapGapAttempts; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up consumes + // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head + // frame was sent": onClose's poison-strike gate and handleServerRejection's + // pre-send gate key off THIS instead. Without it, a transient outage AFTER the + // catch-up but BEFORE the first data frame (a flapping LB/middlebox that accepts + // the upgrade + catch-up then closes) would be mistaken for a deterministic + // head-frame rejection and escalate to a PROTOCOL_VIOLATION terminal -- breaking + // the store-and-forward "retry a transient outage forever" contract. Reset per + // connection in setWireBaselineWithCatchUp; set in trySendOne after a successful + // send. + private boolean dataFrameSentThisConnection; + // End position (native address) written by the last readVarintAt() call. + private long varintEnd; private WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 50fc18ff..602c9ed2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -606,6 +606,62 @@ public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMas return maxEnd; } + /** + * Highest {@code deltaStart} any delta-flagged frame at or below + * {@code maxFsnInclusive} carries, or {@code 0} when NO delta-flagged frame is + * present. A result of {@code 0} means every surviving symbol-bearing frame is + * SELF-SUFFICIENT -- it re-registers its dictionary from id 0 (a full-dict + * frame), so the slot replays with no persisted dictionary at all; a result + * {@code > 0} means at least one frame is a non-self-sufficient delta whose ids + * depend on a prior frame's (or the persisted dictionary's) registrations. + * {@link CursorSendEngine} pairs this with {@link #maxSymbolDeltaEnd} at + * recovery: when the persisted {@code .symbol-dict} is a subset of the ids the + * frames reference BUT this returns {@code 0}, the slot was written in full-dict + * fallback (the dictionary never opened when writing) and recovers in full-dict + * mode instead of failing clean; a torn DELTA dictionary has a {@code + * deltaStart > 0} frame and still fails clean. Same read-only frame walk and + * layout as {@link #maxSymbolDeltaEnd}; only frames at or below {@code + * maxFsnInclusive} count (the retired orphan-deferred tail is excluded). + */ + public long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { + long maxStart = 0L; + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + long fsn = baseSeq + i; + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + long payload = mmapAddress + off + FRAME_HEADER_SIZE; + // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never + // sent, so its baseline must not count. off still advances below. + if (fsn <= maxFsnInclusive + && payloadLen >= qwpHeaderSize + && payloadLen > flagsOffset + && Unsafe.getUnsafe().getInt(payload) == headerMagic + && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { + long p = payload + qwpHeaderSize; + long limit = payload + payloadLen; + long deltaStart = 0L; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaStart |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; // corrupt run; recovered frames are CRC-valid, so defensive only + } + } + if (deltaStart > maxStart) { + maxStart = deltaStart; + } + } + off += FRAME_HEADER_SIZE + payloadLen; + } + return maxStart; + } + /** * Number of frames written since {@link #create} (or recovered by * {@link #openExisting}). Used by {@code SegmentRing} to compute diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 21cfa9d1..f9cd08c6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -566,6 +566,25 @@ public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int return Math.max(maxEnd, end); } + /** + * Highest {@code deltaStart} any symbol-dict delta frame at or below + * {@code maxFsnInclusive} carries (0 when none, or when every such frame is + * self-sufficient). See {@link MmapSegment#maxSymbolDeltaStart}; paired with + * {@link #maxSymbolDeltaEnd} at recovery to tell a full-dict-fallback slot + * (recoverable with no dictionary) from a torn delta dictionary (fails clean). + */ + public synchronized long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { + long maxStart = 0L; + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + long start = sealedSegments.get(i).maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); + if (start > maxStart) { + maxStart = start; + } + } + long start = active.maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); + return Math.max(maxStart, start); + } + public MmapSegment getActive() { return active; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index fb355b82..e576902b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -897,6 +897,87 @@ public void testTornDictTotalLossFailsCleanOnResume() throws Exception { }); } + @Test + public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Exception { + // M1 regression (counterpoint to testTornDictTotalLossFailsCleanOnResume): a + // slot written in FULL-DICT fallback -- the .symbol-dict could not open when + // writing, so isDeltaDictEnabled() was false and every frame re-ships the + // whole dictionary from id 0 (deltaStart=0) -- leaves SELF-SUFFICIENT frames + // and NO side-file. On recovery the engine opens a FRESH EMPTY .symbol-dict, + // so the surviving frames out-reach it (recoveredMaxSymbolId >= pd.size()==0). + // Those frames carry their whole dictionary inline and need no side-file, so + // they must RECOVER, not brick build(). CursorSendEngine detects them + // (maxSymbolDeltaStart == 0) and discards the empty side-file, recovering the + // slot in full-dict mode. Before the fix the sender's seed-time guard treated + // the empty dictionary as a host-crash tear and threw from Sender.build(), + // even though the orphan drainer drains the very same frames fine. A torn + // DELTA dictionary (deltaStart > 0) still fails clean -- see that other test. + assertMemoryLeak(() -> { + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + // Force full-dict fallback in phase 1: plant a non-empty DIRECTORY where + // the dictionary file belongs, so openCleanRW fails and delta encoding + // stays disabled -- the frames are then written self-sufficient. + java.nio.file.Files.createDirectories(dict); + java.nio.file.Path blocker = dict.resolve("blocker"); + java.nio.file.Files.createFile(blocker); + + // Phase 1: silent server (no acks). Sender 1 writes new-symbol rows in + // full-dict mode and close-fast, leaving unacked self-sufficient frames. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + // The planted directory is untouched -- the dictionary never opened, + // so the frames were written full-dict with no side-file. + Assert.assertTrue("full-dict fallback: .symbol-dict must stay a directory", + java.nio.file.Files.isDirectory(dict)); + } + + // Drop the planted directory so recovery opens a FRESH EMPTY .symbol-dict + // where it belongs -- exactly the state a full-dict-fallback slot recovers + // into (frames on disk, no dictionary behind them). + java.nio.file.Files.delete(blocker); + java.nio.file.Files.delete(dict); + + // Phase 2: recover. build() must SUCCEED (not throw the torn-dict guard), + // and the self-sufficient frames replay against a fresh server gap-free. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { // must NOT throw + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } + } + // Full-dict recovery re-ships the whole dictionary inline on every + // frame, so there is NO 0-table catch-up frame (that is the delta-mode + // reconnect path). The reconstructed dictionary is still gap-free. + Assert.assertFalse("full-dict recovery must NOT send a catch-up frame", + handler.sawCatchUpFrame); + List reconstructed = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, reconstructed.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, reconstructed.get(i)); + } + } + }); + } + private static int globalDictSize(Sender sender) throws Exception { Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); f.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index c21b7d82..ba81775f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -528,7 +528,7 @@ private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient } private CursorSendEngine newEngine() { - return new CursorSendEngine(tmpDir, 16384); + return new CursorSendEngine(tmpDir, 16_384); } private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception { From c181e4fa31f915387713ff24076641bd491a8f12 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 19:49:00 +0100 Subject: [PATCH 54/78] Recover slots whose symbol dictionary is damaged The QWP store-and-forward recovery path treated a damaged .symbol-dict as unrecoverable in two places, stranding slots that are in fact replayable. Both stem from keying on the dictionary FILE rather than on what the surviving frames themselves define. CursorWebSocketSendLoop ran the torn-dictionary guard unconditionally but gated accumulateSentDict, the reconnect catch-up and the mirror seed on engine.isDeltaDictEnabled(). That flag goes false exactly when a disk slot's dictionary fails to open -- fd exhaustion, a read-only remount, ENOSPC -- which is precisely when the frames already on disk are still deltas. With the mirror gated off, sentDictCount froze at 0 while the ungated guard kept comparing against it, so the frame at deltaStart=1 latched a terminal and the background drainer quarantined the slot behind a .failed sentinel. Yet replaying that sequence from id 0 rebuilds the dictionary on the server contiguously and drains perfectly. The loop now always mirrors, always catches up and always guards: the mirror is its model of what the SERVER holds, so it must track every mode. isDeltaDictEnabled() now governs only what the producer emits and persists. seedGlobalDictionaryFromPersisted threw whenever the surviving frames out-reached the persisted dictionary. That bricked Sender.build() for slots the drainer replays fine, and permanently: build()'s catch releases the slot lock, but the sender that would have hosted the orphan drainer is the one that just failed, so the retry recovers the same slot and throws again. Relaxing the guard would have been worse than the bug -- with an empty dictionary the producer hands its next symbol id 0, an id the frames already define, and accumulateSentDict skips it, so the next catch-up replays the stale symbol and misattributes it silently. The producer now seeds from the dictionary's intact prefix AND THEN from the surviving frames' own delta sections, through collectReplaySymbolsAbove (MmapSegment -> SegmentRing -> CursorSendEngine). Those are the same two sources, in the same order, that the send loop's mirror is built from, so the producer's delta baseline and the mirror's coverage land on the same number by construction. A genuine gap -- ids introduced by frames since acked and trimmed away -- still fails clean, and now on exactly the condition the send loop's replay guard trips on, so producer and drainer agree on which slots are recoverable. The two cap-gap settle-budget tests derived their retriable loop bound from MAX_CATCHUP_CAP_GAP_ATTEMPTS itself, so a regression of that constant to 1 -- the very bug they name -- ran the loop zero times and let the test pass green. They now pin the budget against a literal. Tests: an untrimmed unopenable-dictionary slot must replay gap-free; a torn (subset) and a wholly lost dictionary must rebuild from the surviving frames and place a new symbol above the recovered tip. The two watermark-stamped siblings still fail clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 151 +++++----- .../client/sf/cursor/CursorSendEngine.java | 38 +++ .../sf/cursor/CursorWebSocketSendLoop.java | 159 ++++++----- .../qwp/client/sf/cursor/MmapSegment.java | 117 ++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 35 +++ .../qwp/client/DeltaDictRecoveryTest.java | 263 +++++++++++++----- ...WebSocketSendLoopCatchUpAlignmentTest.java | 20 ++ 7 files changed, 573 insertions(+), 210 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c9f7203c..bc08948e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3847,91 +3847,88 @@ private void resetSymbolDictStateForNewConnection() { } /** - * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} from - * the slot's persisted dictionary (ids assigned in the same ascending order, - * so they match the recovered frames) and resumes the delta baseline at the - * recovered tip, so newly ingested symbols continue above the recovered ids. + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} so that newly + * ingested symbols continue ABOVE every id the surviving frames already define, and + * resumes the delta baseline at that tip. *

- * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): - * the persisted dictionary, the on-wire delta and the send-loop catch-up mirror - * all key on the entry POSITION (id), so the producer id space must match the - * persisted entry count exactly. {@code getOrAddSymbol} would collapse two - * source strings that decode to the same characters -- only malformed lone - * UTF-16 surrogates, which UTF-8-encode to {@code '?'} -- leaving this - * dictionary shorter than {@code pd.size()} and desyncing - * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()}, - * which silently misattributes later symbols after a reconnect. + * Seeds from TWO sources, in this order: + *

    + *
  1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
    2. + *
  2. the surviving frames' OWN delta sections, for every id above that prefix + * ({@link CursorSendEngine#collectReplaySymbolsAbove}).
    3. + *
+ * Those are exactly the two sources, in exactly the order, that the send loop's mirror + * is built from: its constructor seeds {@code sentDictCount} from the same dictionary, + * and {@code accumulateSentDict} then extends it from the same frames as they replay. So + * the producer's {@code sentMaxSymbolId + 1} and the loop's {@code sentDictCount} land on + * the same number BY CONSTRUCTION -- the invariant the torn-dictionary guard rests on -- + * rather than by the two happening to agree. *

- * Host-crash tear guard. The persisted dictionary is NOT fsync'd (see - * {@code PersistedSymbolDict}), so a host/power crash can lose its - * most-recently-written (highest-id) entries while the segment frames that - * introduced those ids survive -- and those newest frames, being the least - * likely to be acked, replay on recovery. The send loop's catch-up mirror then - * rebuilds the missing ids from those frames' own delta bytes, but THIS producer - * -- seeded only from the shorter dictionary -- would assign its next new symbol - * an id the surviving frames already define, putting two symbols on one id and - * silently misattributing values. The send loop's replay guard only catches a - * GAP ({@code deltaStart > sentDictCount}); a frame that introduces exactly the - * torn-off id ({@code deltaStart == pd.size()}) slips through and self-heals the - * mirror, leaving only this producer diverged. Detect it here -- the surviving - * frames reference an id at or beyond the recovered dictionary size -- and fail - * clean: the affected data must be resent, matching the design's torn-dict - * "resend required" contract. + * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): the + * persisted dictionary, the on-wire delta and the mirror all key on the entry POSITION + * (id), so the producer's id space must match the recovered entry count exactly. + * {@code getOrAddSymbol} would collapse two source strings that decode to the same + * characters -- only malformed lone UTF-16 surrogates, which UTF-8-encode to {@code '?'} + * -- leaving this dictionary SHORTER than the count and silently misattributing later + * symbols. *

- * The background drainer self-heals the mirror ONLY when a surviving frame - * STRADDLES the tear ({@code deltaStart <= pd.size() < deltaStart + deltaCount}): - * such a frame carries the torn-off ids in its own delta and - * {@code accumulateSentDict} re-registers them, so the drainer drains the slot. - * But when the symbol-introducing frames were already acked and trimmed and only - * a HIGHER-baseline frame survives ({@code deltaStart > pd.size()} -- e.g. a - * commit or a symbol-reusing frame, since {@code beginMessage} always sets the - * delta flag), the drainer's own replay guard ({@code deltaStart > sentDictCount}) - * fires too and quarantines the slot: the recorded bytes are not silently lost, - * but the slot is NOT auto-drained -- it must be resent. That is a deliberate - * CONSERVATIVE over-strand -- the guard keys on {@code deltaStart}, not on the - * frame's actual highest referenced id, to avoid parsing row data at recovery, - * so it may reject a frame whose rows reference only ids the truncated - * dictionary still holds. It fails clean rather than risk a silent id shift. - * Only a host crash reaches this -- a process crash keeps the page cache, so the - * write-ahead ordering keeps the dictionary a superset of the frames. + * Why seeding from the frames matters. The dictionary is not fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can tear off its newest entries + * while the segment frames that introduced those ids survive -- and those newest frames, + * being the least likely to be acked, are exactly the ones that replay. Seeded from the + * short dictionary alone, this producer would hand its next new symbol an id those frames + * already define, putting two symbols on one id and silently misattributing values. The + * old code detected that and threw, which was safe but far too blunt: it bricked + * {@code build()} for slots the background drainer replays PERFECTLY, because the frames + * carry the torn-off symbols in their own deltas and {@code accumulateSentDict} rebuilds + * the dictionary from them. This method now rebuilds the producer from the same bytes, + * so a torn -- or entirely lost -- dictionary is recoverable whenever the surviving + * frames define the ids themselves. The next flush's write-ahead persist then re-writes + * those ids (it resumes from {@code pd.size()}), healing the side-file on disk. + *

+ * What still fails clean. A genuine GAP: the ids below a surviving frame's delta + * start were introduced by frames that were acked and TRIMMED away, so they lived only in + * the lost dictionary and nothing can rebuild them. + * {@code collectReplaySymbolsAbove} returns -1 for that and we throw. It is the same + * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so + * producer and drainer now agree on exactly which slots are recoverable, instead of the + * producer rejecting slots the drainer drains. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { - if (pd == null) { + if (cursorEngine == null) { return; } - // Run the torn-dictionary guard BEFORE the empty-dictionary short-circuit - // below: a TOTAL tear (pd.size() == 0) of a DELTA dictionary with surviving - // symbol-bearing frames must fail clean too, not slip through. Such a frame - // starts at deltaStart=0 and self-heals the I/O-thread catch-up mirror, so the - // send loop's replay guard (deltaStart > sentDictCount) never fires -- this - // seed-time guard is then the only defense against the producer resuming - // unseeded and silently reusing ids the frames already define. A genuinely - // empty slot (no symbol-bearing frames) has recoveredMaxSymbolId() == -1, so - // -1 >= 0 is false and the pd.size() == 0 return below still fires. - // - // This guard fires ONLY for a torn DELTA dictionary. The other way the frames - // can out-reach the recovered dictionary -- a slot written in FULL-DICT - // fallback (the dictionary never opened when writing, every frame - // self-sufficient at deltaStart=0), then recovered against a fresh empty one -- - // is caught upstream in CursorSendEngine, which discards the empty side-file so - // isDeltaDictEnabled() is false and this seed never runs. Those frames need no - // dictionary; failing clean here would needlessly brick build() for a slot the - // orphan drainer drains fine. - if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { + // 1. The dictionary's intact prefix. addRecoveredSymbol appends without de-dup, so + // the producer's size tracks pd.size() exactly -- which is what the send loop's + // mirror also seeds sentDictCount from. + int baseline = 0; + if (pd != null && pd.size() > 0) { + ObjList persisted = pd.readLoadedSymbols(); + for (int i = 0, n = persisted.size(); i < n; i++) { + globalSymbolDictionary.addRecoveredSymbol(persisted.getQuick(i)); + } + baseline = globalSymbolDictionary.size(); + } + // 2. Everything the surviving frames define above that prefix, straight out of their + // own delta sections -- the same bytes, in the same order, accumulateSentDict will + // feed the mirror as those frames go back on the wire. + ObjList fromFrames = new ObjList<>(); + long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames); + if (coverage < 0) { throw new LineSenderException( - "recovered store-and-forward symbol dictionary is a subset of the surviving frames " - + "(likely a host crash tore its unsynced tail): frames reference symbol id " - + cursorEngine.recoveredMaxSymbolId() + " but the recovered dictionary holds only " - + pd.size() + " id(s); resuming would reuse ids the frames already define -- " - + "resend the affected data"); - } - if (pd.size() == 0) { - return; - } - ObjList symbols = pd.readLoadedSymbols(); - for (int i = 0, n = symbols.size(); i < n; i++) { - globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); - } + "recovered store-and-forward symbol dictionary is incomplete and cannot be " + + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + + "tail): the frames reference symbol ids below their own delta start, which " + + "were introduced by frames since acked and trimmed away, so nothing still " + + "holds them; the recovered dictionary holds only " + + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data"); + } + for (int i = 0, n = fromFrames.size(); i < n; i++) { + globalSymbolDictionary.addRecoveredSymbol(fromFrames.getQuick(i)); + } + // Producer baseline == the coverage the replay will establish == the mirror's + // sentDictCount once those frames have gone out. The first new frame therefore + // starts its delta exactly at the tip, and the replay guard passes. sentMaxSymbolId = globalSymbolDictionary.size() - 1; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 88a59735..91035a0e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -661,6 +661,44 @@ public synchronized void close() { } } + /** + * Rebuilds, from the surviving frames' OWN delta sections, the symbols the upcoming + * replay will register ABOVE {@code baseline}, appending them to {@code out} in + * ascending id order. Returns the coverage the replay establishes (one past the highest + * id it registers), or {@code -1} when the frames have a genuine GAP above the baseline. + *

+ * The producer seeds its dictionary from the persisted {@code .symbol-dict} and THEN + * from this, so it is fed the same entries, in the same order, that the send loop's + * {@code accumulateSentDict} will feed its mirror from as those very frames replay. The + * producer's delta baseline and the loop's mirror coverage therefore land on the same + * number by construction, which is the invariant the torn-dictionary guard rests on. + *

+ * This is what makes a slot whose dictionary was torn -- or lost outright -- still + * recoverable when the surviving frames define the ids themselves (they carry the + * symbols in their own deltas; it is why the orphan drainer can drain such a slot). Only + * a real gap, where the ids were introduced by frames since acked and trimmed away, is + * unrecoverable -- and that is precisely when this returns -1. + *

+ * Bounded above by {@link #recoveredCommitBoundaryFsn} like {@link #recoveredMaxSymbolId}: + * frames past it are the aborted orphan-deferred tail, retired without ever being + * transmitted, so their ids never reach a server and must not inflate the baseline. + */ + public long collectReplaySymbolsAbove(int baseline, ObjList out) { + if (ring == null) { + return baseline; + } + return ring.collectReplaySymbolsAbove( + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT, + QwpConstants.HEADER_SIZE, + ackedFsn() + 1L, + recoveredCommitBoundaryFsn, + baseline, + out + ); + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 0c17614f..93b0b137 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -168,11 +168,6 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; - // True when this loop delta-encodes symbol dictionaries: it keeps a reconnect - // catch-up mirror (sentDict*) and re-registers the whole dictionary on a fresh - // server before replaying delta frames. See the sentDict* fields in the mutable - // section for the full mechanism. Gates all the delta-dict state. - private final boolean deltaDictEnabled; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -239,17 +234,21 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); - // Delta symbol dictionary catch-up state (see swapClient, deltaDictEnabled). - // Active in memory mode, and in disk mode whenever the per-slot persisted - // dictionary opened -- fresh slots included, not just recovered / orphan-drained - // ones. On a recovered / orphan-drained slot the constructor additionally SEEDS - // sentDict* from that persisted dictionary; a fresh slot starts with an empty - // mirror and grows it as frames are sent. The loop mirrors, in sentDict*, every - // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in - // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that on - // reconnect it can re-register the whole dictionary on the fresh server (which - // discards its dictionary on every disconnect) before replaying frames whose - // deltas start above id 0. All of this is touched only by the I/O thread. + // Delta symbol dictionary catch-up state (see swapClient). + // ALWAYS active -- in memory mode, in disk mode, and (critically) even when the + // per-slot persisted dictionary failed to open. sentDictCount is this loop's model + // of how many ids the CURRENT server has been told about, and the torn-dict guard + // in trySendOne reads it, so it must track the wire in every mode; gating it on + // engine.isDeltaDictEnabled() is what once froze it at 0 and terminal'd a slot + // whose frames replay perfectly from id 0 (see the constructor). On a recovered / + // orphan-drained slot the constructor SEEDS sentDict* from the persisted + // dictionary when there is one; otherwise the mirror starts empty and grows from + // the frames themselves. The loop mirrors, in sentDict*, every symbol it has ever + // sent -- the concatenated [len varint][utf8] bytes in global-id order + // (sentDictBytes*) plus the count (sentDictCount) -- so that on reconnect it can + // re-register the whole dictionary on the fresh server (which discards its + // dictionary on every disconnect) before replaying frames whose deltas start above + // id 0. All of this is touched only by the I/O thread. // Footprint note: this mirror is a SECOND copy of the dictionary -- the same // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a @@ -571,39 +570,50 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; - this.deltaDictEnabled = engine.isDeltaDictEnabled(); // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, // so seed it from the slot's persisted dictionary. That way the very first // connection re-registers the whole dictionary (via a catch-up frame) // before replaying the recovered delta frames. - if (deltaDictEnabled) { - PersistedSymbolDict pd = engine.getPersistedSymbolDict(); - if (pd != null && pd.size() > 0) { - int len = pd.loadedEntriesLen(); - if (len > 0) { - // COPY the persisted dictionary's loaded-entries buffer into this - // loop's own mirror rather than taking ownership of it. The engine - // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan - // drainer path: BackgroundDrainer builds a fresh send loop per wire - // session against the same engine on a durable-ack capability-gap - // recycle. A one-shot ownership transfer would leave every loop - // after the first with an EMPTY mirror -- it would then send no - // reconnect catch-up, and the first replayed delta frame - // (deltaStart > 0) would trip the torn-dict guard, falsely - // quarantining a healthy slot. Copying keeps the dictionary's - // loaded entries intact for the engine's lifetime so every - // recycled loop re-seeds; pd.close() (at engine close) frees the - // dictionary's copy, this loop frees its own copy on exit. - sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); - sentDictBytesCapacity = len; - sentDictBytesLen = len; - // Set the count only alongside the bytes so sentDictCount can - // never claim symbols the mirror does not hold. A recovered slot - // always has loadedEntriesLen > 0 when size > 0, so this is the - // same result -- it just makes the coupling explicit. - sentDictCount = pd.size(); - } + // + // Deliberately NOT gated on engine.isDeltaDictEnabled(). The mirror, the + // catch-up and the replay guard together are this loop's model of what the + // SERVER's dictionary holds, and that model must track every mode. Gating + // them on that flag is what once bricked a recoverable slot: an unopenable + // .symbol-dict reports isDeltaDictEnabled()=false, yet the frames already on + // disk are still DELTA frames (deltaStart 0,1,2,...). With the mirror gated + // off, sentDictCount froze at 0 while the ungated guard kept comparing + // against it, so the frame at deltaStart=1 tripped a terminal -- even though + // replaying the whole sequence from id 0 rebuilds the dictionary on the + // server contiguously and drains perfectly. isDeltaDictEnabled() decides + // what the PRODUCER emits and persists; it must never decide what this loop + // tracks. When the dictionary could not be opened, pd is null, the mirror + // simply starts empty and grows from the frames themselves. + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + if (pd != null && pd.size() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + // COPY the persisted dictionary's loaded-entries buffer into this + // loop's own mirror rather than taking ownership of it. The engine + // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan + // drainer path: BackgroundDrainer builds a fresh send loop per wire + // session against the same engine on a durable-ack capability-gap + // recycle. A one-shot ownership transfer would leave every loop + // after the first with an EMPTY mirror -- it would then send no + // reconnect catch-up, and the first replayed delta frame + // (deltaStart > 0) would trip the torn-dict guard, falsely + // quarantining a healthy slot. Copying keeps the dictionary's + // loaded entries intact for the engine's lifetime so every + // recycled loop re-seeds; pd.close() (at engine close) frees the + // dictionary's copy, this loop frees its own copy on exit. + sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + sentDictBytesCapacity = len; + sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold. A recovered slot + // always has loadedEntriesLen > 0 when size > 0, so this is the + // same result -- it just makes the coupling explicit. + sentDictCount = pd.size(); } } this.fsnAtZero = fsnAtZero; @@ -2055,7 +2065,15 @@ private void setWireBaselineWithCatchUp(long replayStart) { // handleServerRejection can tell "only the catch-up went out" from "the // head data frame went out". dataFrameSentThisConnection = false; - if (client != null && deltaDictEnabled && sentDictCount > 0) { + // Not gated on isDeltaDictEnabled() -- see the constructor. The mirror is + // non-empty exactly when this loop has already put symbols on a wire, and a + // FRESH server holds none of them, so it must be re-registered before any + // frame that back-references an id. In full-dict mode the mirror is non-empty + // too and the catch-up is then redundant (the next frame re-registers from id + // 0 anyway) but harmless: the server overwrites identical ids with identical + // symbols. Redundant-but-uniform beats a mode flag that can go false while the + // frames on disk are still deltas. + if (client != null && sentDictCount > 0) { this.nextWireSeq = 0L; // The catch-up may span several frames when the dictionary exceeds the // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps @@ -2493,24 +2511,23 @@ private boolean trySendOne() { return false; // payload not fully published yet } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; - // Torn-dictionary guard. Decode the delta start unconditionally (-1 for a - // non-delta frame); the guard MUST run even when deltaDictEnabled is false. - // A disk slot recovered with its persisted dictionary unavailable - // (PersistedSymbolDict.open() returned null -- fd exhaustion, a read-only - // remount, ENOSPC) reports deltaDictEnabled=false, yet its recorded frames - // are still DELTA frames (deltaStart > 0). Replaying those against a fresh - // empty-dictionary server would null-pad the missing ids and SILENTLY - // corrupt the table -- precisely what this guard exists to prevent -- so it - // cannot be gated on the very flag that goes false in that failure mode. In - // normal operation a delta frame's start id never exceeds the dictionary - // coverage established so far (replayed frames overlap the catch-up dict; - // fresh frames extend it contiguously), so a gap here means the recovered - // dictionary is incomplete (a host/power crash that lost recently-written - // entries, SF being process-crash but not host-crash durable). Fail - // terminally; the unreplayable data must be resent. Full-dict / fallback - // frames carry deltaStart=0 with sentDictCount=0, so 0 > 0 never - // false-positives; only the sent-dictionary mirror below stays gated on - // deltaDictEnabled. + // Torn-dictionary guard. sentDictCount is this loop's model of how many ids + // the CURRENT server has been told about: seeded from the persisted + // dictionary, re-registered by the catch-up, and extended by + // accumulateSentDict as each frame goes out. A frame whose delta starts ABOVE + // that coverage references ids the server was never given; replaying it would + // make the server null-pad the hole (QwpMessageCursor grows the dict with + // nulls to deltaStart+deltaCount) and land rows with SILENTLY NULL symbols. + // Fail terminally instead; the unreplayable data must be resent. + // + // The guard, the mirror and the catch-up are ONE mechanism and are all + // ungated (see the constructor). A frame at deltaStart == sentDictCount is + // contiguous and extends the coverage, so a slot whose frames start at id 0 + // replays cleanly with no persisted dictionary at all -- which is exactly why + // the mirror below must keep advancing even when the dictionary is missing. + // A gap here therefore means genuine loss: a host/power crash tore the + // unsynced .symbol-dict below the ids the surviving frames still reference + // (SF is process-crash but not host-crash durable). int deltaStart = frameDeltaStart(frameAddr, payloadLen); if (deltaStart > sentDictCount) { recordFatal(new LineSenderException( @@ -2530,10 +2547,16 @@ private boolean trySendOne() { // key their poison-strike vs pre-send decision off this, not off nextWireSeq // (which the catch-up advances). dataFrameSentThisConnection = true; - if (deltaDictEnabled && deltaStart >= 0) { - // Mirror the symbols this frame introduced so a later reconnect can - // rebuild the whole dictionary. Idempotent on replay: a frame whose - // delta we already hold advances nothing. + if (deltaStart >= 0) { + // Mirror the symbols this frame just registered on the server, so a later + // reconnect can rebuild the whole dictionary and the guard above keeps an + // accurate view of the server's coverage. Idempotent on replay: a frame + // whose delta we already hold advances nothing. + // + // Ungated (see the constructor): this is the ONLY thing that advances + // sentDictCount from the frames themselves, so gating it while leaving the + // guard ungated froze the coverage at 0 and terminal'd a slot that replays + // perfectly from id 0. accumulateSentDict(frameAddr, payloadLen, deltaStart); } lastFrameOrPingNanos = System.nanoTime(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 602c9ed2..5ad281e1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -28,9 +28,11 @@ import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -553,6 +555,121 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in * retired orphan-deferred tail -- pass * {@code recoveredCommitBoundaryFsn} */ + /** + * Rebuilds, from the frames' own delta sections, the symbols a replay of + * {@code [minFsnInclusive .. maxFsnInclusive]} would register ABOVE {@code coverage}, + * appending them to {@code out} in ascending id order. Returns the coverage the walk + * establishes (one past the highest id registered), or {@code -1} when a frame's delta + * starts ABOVE the coverage reached so far. + *

+ * This is the recovery-time twin of {@code CursorWebSocketSendLoop.accumulateSentDict}: + * it visits the same frames, in the same FSN order, and extracts exactly the entries + * that method will add to the send loop's mirror as those frames go back on the wire. + * The producer's dictionary is seeded from it so the producer's delta baseline and the + * loop's mirror coverage stay in lockstep BY CONSTRUCTION -- which is what makes a slot + * whose {@code .symbol-dict} was torn (or lost entirely) recoverable, as long as the + * surviving frames define the ids themselves. + *

+ * A {@code -1} return is a genuine GAP: the ids between the coverage and the frame's + * delta start were introduced by frames that were acked and trimmed away, so they lived + * ONLY in the lost dictionary and cannot be rebuilt from anything. That is the one case + * the data really must be resent, and it is exactly the condition the send loop's + * torn-dictionary guard ({@code deltaStart > sentDictCount}) would trip on at replay -- + * so the two agree, rather than the producer failing on slots the drainer drains fine. + *

+ * Same frame layout and FSN bound as {@link #maxSymbolDeltaEnd}. + */ + public long collectReplaySymbolsAbove( + int headerMagic, + int flagsOffset, + int flagDeltaMask, + int qwpHeaderSize, + long minFsnInclusive, + long maxFsnInclusive, + long coverage, + ObjList out + ) { + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + long fsn = baseSeq + i; + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + long payload = mmapAddress + off + FRAME_HEADER_SIZE; + if (fsn >= minFsnInclusive + && fsn <= maxFsnInclusive + && payloadLen >= qwpHeaderSize + && payloadLen > flagsOffset + && Unsafe.getUnsafe().getInt(payload) == headerMagic + && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { + long p = payload + qwpHeaderSize; + long limit = payload + payloadLen; + long deltaStart = 0L; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaStart |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; // corrupt run; recovered frames are CRC-valid, so defensive only + } + } + long deltaCount = 0L; + shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaCount |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; + } + } + if (deltaStart > coverage) { + return -1L; // gap: ids [coverage, deltaStart) are unrecoverable + } + // Walk this frame's [len varint][utf8] entries. Ids below the coverage we + // already hold are skipped (a replayed frame legitimately overlaps the tip); + // ids at or above it extend the dictionary. + long id = deltaStart; + for (long k = 0; k < deltaCount; k++, id++) { + long len = 0L; + shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + len |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; + } + } + if (p + len > limit) { + // Malformed entry region. Do NOT guess: report it as unrecoverable + // so the caller fails clean rather than seeding a shifted dictionary. + return -1L; + } + if (id >= coverage) { + out.add(Utf8s.stringFromUtf8Bytes(p, p + len)); + } + p += len; + } + long deltaEnd = deltaStart + deltaCount; + if (deltaEnd > coverage) { + coverage = deltaEnd; + } + } + off += FRAME_HEADER_SIZE + payloadLen; + } + return coverage; + } + public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; long off = HEADER_SIZE; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index f9cd08c6..3ec06014 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -554,6 +554,41 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred * tail and are excluded. */ + /** + * Rebuilds the symbols a replay of {@code [minFsnInclusive .. maxFsnInclusive]} would + * register ABOVE {@code baseline}, appending them to {@code out} in ascending id order. + * Returns the coverage the replay establishes, or {@code -1} on a genuine gap. See + * {@link MmapSegment#collectReplaySymbolsAbove}. + *

+ * Walks the segments in ascending FSN order -- {@code sealedSegments} are held in + * baseSeq order and {@code active} is the newest (see {@code recover}) -- threading the + * coverage through, because unlike {@link #maxSymbolDeltaEnd} (a plain max) this walk is + * ORDER-DEPENDENT: each frame may only extend a dictionary the frames before it built. + */ + public synchronized long collectReplaySymbolsAbove( + int headerMagic, + int flagsOffset, + int flagDeltaMask, + int qwpHeaderSize, + long minFsnInclusive, + long maxFsnInclusive, + long baseline, + ObjList out + ) { + long coverage = baseline; + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + coverage = sealedSegments.get(i).collectReplaySymbolsAbove( + headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, + minFsnInclusive, maxFsnInclusive, coverage, out); + if (coverage < 0) { + return -1L; + } + } + return active.collectReplaySymbolsAbove( + headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, + minFsnInclusive, maxFsnInclusive, coverage, out); + } + public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; for (int i = 0, n = sealedSegments.size(); i < n; i++) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e576902b..31153408 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -204,7 +204,7 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception 0, handler.frames.get()); Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("subset of the surviving frames") + terminal.getMessage().contains("cannot be rebuilt from the surviving frames") && terminal.getMessage().contains("resend")); } }); @@ -347,6 +347,106 @@ public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { }); } + @Test + public void testUnopenablePersistedDictReplaysSelfSufficientFrameSequence() throws Exception { + // Untrimmed counterpart of + // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames. + // + // Same failure mode -- the persisted dictionary cannot be OPENED, so + // CursorSendEngine.isDeltaDictEnabled() is false -- but NOTHING was acked, so + // replay starts at the FIRST frame. Frames 0..5 carry deltaStart 0..5 and are + // COLLECTIVELY self-sufficient: replayed in order from id 0, the server + // accumulates the dictionary contiguously, needing no dictionary file at all. + // + // Pre-fix, accumulateSentDict was gated on deltaDictEnabled while the + // torn-dict guard was NOT, so sentDictCount froze at 0: frame 0 (deltaStart=0) + // passed the guard, but frame 1 (deltaStart=1 > 0) tripped it and latched a + // terminal -- and for the background drainer that means markFailed + a .failed + // sentinel, permanently quarantining a slot that drains perfectly. A store- + // and-forward contract violation: a TRANSIENT disk condition (fd exhaustion, a + // read-only remount, ENOSPC) stranding recoverable data. + // + // The trimmed sibling pins the other half of the contract: when replay starts + // ABOVE the frames that introduced the ids, the terminal is still correct. + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i. + // The server never acks, so nothing is trimmed and replay starts at frame 0 + // (the common orphan-drain profile: the server was down the whole time). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Make the persisted dictionary UNOPENABLE: a directory of the same name + // defeats both openRW and openCleanRW, so PersistedSymbolDict.open() + // returns null. Deliberately do NOT stamp the ack watermark -- replay must + // start at frame 0, which is what makes the sequence self-sufficient. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + + // Phase 2: recover against a fresh server that reconstructs its per- + // connection dictionary from the wire exactly as the real one does -- + // null-padding any gap, so a gap surfaces as a null rather than passing + // silently. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < 6 + && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + + // Pre-fix: the "symbol dictionary is incomplete" terminal fires on frame 1. + Assert.assertNull("a self-sufficient frame sequence must replay without a terminal: " + + (terminal == null ? "" : terminal.getMessage()), + terminal); + // Pre-fix: the server dictionary stops at ["sym-0"] -- frame 1 never shipped. + List serverDict = handler.dictSnapshot(); + Assert.assertEquals("every delta frame must replay [dict=" + serverDict + ']', + 6, serverDict.size()); + for (int i = 0; i < 6; i++) { + Assert.assertEquals("id " + i + " must resolve; a null here is a server-side " + + "null-pad, i.e. a silently NULL symbol column [dict=" + serverDict + ']', + "sym-" + i, serverDict.get(i)); + } + } + }); + } + @Test public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { // C1 regression: when a recovered disk slot's persisted dictionary cannot be @@ -760,23 +860,29 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti } @Test - public void testTornDictSubsetFailsCleanOnResume() throws Exception { - // C1 regression: the persisted .symbol-dict is NOT fsync'd, so a host crash - // can lose its highest-id tail entries while the segment frames that - // introduced those ids survive (out-of-order page loss). Recovery then seeds - // the producer from the SHORTER dictionary, but the I/O-thread catch-up mirror - // rebuilds the missing ids from those surviving frames -- so a resumed producer - // would assign its next new symbol an id the frames already define, silently - // misattributing symbol values on the wire. The send loop's replay guard only - // catches a GAP (deltaStart > mirror); a frame introducing exactly the - // torn-off id slips through and self-heals the mirror, leaving only this - // producer diverged. seedGlobalDictionaryFromPersisted must detect that the - // surviving frames reference an id at/beyond the recovered dictionary size and - // fail clean. Without the fix the resume succeeds and the reuse corrupts - // silently -- so the build below would NOT throw and the test's fail() fires. + public void testTornDictSubsetRebuildsFromSurvivingFrames() throws Exception { + // The persisted .symbol-dict is NOT fsync'd, so a host crash can lose its + // highest-id tail entries while the segment frames that introduced those ids + // survive (out-of-order page loss). Those frames carry the torn-off symbols in + // their OWN delta sections -- which is exactly why the background drainer replays + // such a slot perfectly: accumulateSentDict rebuilds the dictionary from them. + // + // The producer must do the same. Pre-fix it seeded ONLY from the short dictionary, + // saw that the frames out-reached it, and threw -- permanently bricking + // Sender.build() for a slot the drainer drains fine. And the brick is permanent: + // build()'s catch releases the slot lock, but the sender that would have hosted the + // orphan drainer is the one that just failed, so nothing drains the slot and the + // retry recovers it and throws again. seedGlobalDictionaryFromPersisted now seeds + // from the dictionary's intact prefix AND THEN from the surviving frames' deltas, + // so the producer's baseline lands on exactly the coverage the send loop's mirror + // will reach. + // + // testTornDictionaryFailsCleanlyInsteadOfCorrupting pins the other half of the + // contract: when the symbol-introducing frames were acked and TRIMMED away, the ids + // really are unrecoverable and the resume must still fail clean. assertMemoryLeak(() -> { - // Phase 1: record three delta frames (a@0, b@1, c@2) to disk with nothing - // acked (silent server), so all three survive and replay on recovery. + // Phase 1: three delta frames (a@0, b@1, c@2) with nothing acked, so all three + // survive and replay from frame 0. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); silent.start(); @@ -796,10 +902,10 @@ public void testTornDictSubsetFailsCleanOnResume() throws Exception { } } - // Simulate the host-crash tear: rewrite the persisted dictionary to drop - // its highest entry (c@2), keeping a@0,b@1 -- while the frame referencing - // c@2 stays on disk. openClean truncates; the two appends leave the exact - // two-entry dictionary a torn tail would recover to. + // Simulate the host-crash tear: rewrite the dictionary to drop its highest + // entry (c@2), keeping a@0,b@1 -- while the frame that introduced c@2 stays on + // disk. openClean truncates; the two appends leave exactly the two-entry + // dictionary a torn tail would recover to. String slotDir = Paths.get(sfDir, "default").toString(); try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { Assert.assertNotNull(torn); @@ -808,44 +914,58 @@ public void testTornDictSubsetFailsCleanOnResume() throws Exception { Assert.assertEquals(2, torn.size()); } - // Phase 2: a resuming sender must refuse -- the surviving frames reference - // id 2 but the recovered dictionary holds only 2 id(s) [0,1]. - try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + // Phase 2: the resuming sender must REBUILD c@2 from that surviving frame, + // replay everything, and keep ingesting ABOVE the recovered tip. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); good.start(); Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try { - Sender resumed = Sender.fromConfig(cfg); - resumed.close(); - Assert.fail("resume on a torn (subset) dictionary must fail clean"); - } catch (LineSenderException expected) { - Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), - expected.getMessage().contains("subset of the surviving frames") - && expected.getMessage().contains("resend")); + // Pre-fix this line THREW ("subset of the surviving frames"). + try (Sender resumed = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 3) { + Thread.sleep(20); + } + // A NEW symbol must land ABOVE the recovered tip. Seeded short, the + // producer would hand "d" an id the surviving frames already define, + // silently overwriting that symbol on the server -- the corruption the + // old throw existed to prevent, and which this seeding removes at the + // source rather than by refusing to start. + resumed.table("m").symbol("s", "d").longColumn("v", 3).atNow(); + resumed.flush(); + deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 4) { + Thread.sleep(20); + } } + + List dict = handler.dictSnapshot(); + Assert.assertEquals("dictionary must rebuild gap-free [" + dict + ']', 4, dict.size()); + Assert.assertEquals("a", dict.get(0)); + Assert.assertEquals("b", dict.get(1)); + // Rebuilt from the surviving frame's own delta, not from the torn file. + Assert.assertEquals("c", dict.get(2)); + // The new symbol, placed above the recovered tip -- no id reuse. + Assert.assertEquals("d", dict.get(3)); } }); } @Test - public void testTornDictTotalLossFailsCleanOnResume() throws Exception { - // C1 regression (TOTAL-loss variant of testTornDictSubsetFailsCleanOnResume): - // a host crash can lose the ENTIRE persisted .symbol-dict (size 0) while the - // frames that introduced its ids survive; equivalently, a prior session whose - // openClean failed ran full-dict mode and never wrote the side-file, so this - // session recovers those frames against a FRESH EMPTY dictionary. The recovered - // frames start at deltaStart=0 and self-heal the I/O-thread catch-up mirror, so - // the send loop's replay guard (deltaStart > mirror) never fires -- only - // seedGlobalDictionaryFromPersisted can catch it. Its pd.size()==0 early return - // must NOT skip the torn-dict guard: the surviving frames reference id 2 while - // the recovered dictionary holds 0 id(s), so resuming unseeded would reuse those - // ids and silently misattribute symbol values on the wire. Without the fix the - // build below does NOT throw (the guard is bypassed) and the test's fail() fires. + public void testTornDictTotalLossRebuildsFromSurvivingFrames() throws Exception { + // The total-loss counterpart of testTornDictSubsetRebuildsFromSurvivingFrames: the + // whole dictionary is gone (size 0) but the file still opens, so + // isDeltaDictEnabled() stays true and the engine does NOT discard it (the frames + // carry deltaStart 0,1,2, so maxSymbolDeltaStart != 0 and the full-dict-fallback + // discard correctly stays out of the way). + // + // The surviving frames start at id 0 and are collectively self-sufficient, so the + // producer can rebuild the ENTIRE dictionary from their deltas -- exactly as the + // send loop's mirror does. Pre-fix this threw and bricked build(). assertMemoryLeak(() -> { - // Phase 1: record three delta frames (a@0, b@1, c@2) with nothing acked, so - // all three survive and replay -- from FSN 0 (deltaStart=0), the case the - // I/O-thread guard self-heals rather than rejects. + // Phase 1: a@0, b@1, c@2; nothing acked, so all three replay from frame 0. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); silent.start(); @@ -865,35 +985,48 @@ public void testTornDictTotalLossFailsCleanOnResume() throws Exception { } } - // Total dictionary loss: openClean truncates .symbol-dict to its bare header - // (size 0) with zero appends, while every frame -- including the ones - // referencing a@0,b@1,c@2 -- stays on disk. This is the fresh-empty-dict - // state a full-dict-mode prior session or a total host-crash tear leaves. + // Total tear: the dictionary recovers EMPTY but still opens. String slotDir = Paths.get(sfDir, "default").toString(); try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { Assert.assertNotNull(torn); Assert.assertEquals(0, torn.size()); } - // Phase 2: a resuming sender must refuse at build -- the surviving frames - // reference id 2 but the recovered dictionary holds 0 id(s). The I/O-thread - // guard cannot help (deltaStart 0 self-heals the mirror), so the seed-time - // guard is the sole defense. - try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); good.start(); Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try { - Sender resumed = Sender.fromConfig(cfg); - resumed.close(); - Assert.fail("resume on a totally lost (size 0) dictionary must fail clean"); - } catch (LineSenderException expected) { - Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), - expected.getMessage().contains("subset of the surviving frames") - && expected.getMessage().contains("resend")); + // Pre-fix this line THREW. + try (Sender resumed = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 3) { + Thread.sleep(20); + } + resumed.table("m").symbol("s", "d").longColumn("v", 3).atNow(); + resumed.flush(); + deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 4) { + Thread.sleep(20); + } } + + List dict = handler.dictSnapshot(); + Assert.assertEquals("the whole dictionary must rebuild from the frames [" + dict + ']', + 4, dict.size()); + Assert.assertEquals("a", dict.get(0)); + Assert.assertEquals("b", dict.get(1)); + Assert.assertEquals("c", dict.get(2)); + Assert.assertEquals("d", dict.get(3)); } + // The side-file is healed in passing -- persistNewSymbolsBeforePublish resumes + // from pd.size() (0 here), so the flush above re-persisted the rebuilt ids + // a,b,c along with d. That is not asserted on disk because it is not + // observable here: the slot drains fully, and a fully-drained close removes + // the dictionary (CursorSendEngine -> PersistedSymbolDict.removeOrphan), so + // re-opening it would just yield a fresh empty file. The persist-resumes-from- + // pd.size() behaviour is pinned by the testFailedPublish* tests. }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index ba81775f..49411275 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -280,6 +280,19 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); maxField.setAccessible(true); int maxAttempts = maxField.getInt(null); + // Pin the budget against a LITERAL before deriving anything from it. The + // retriable loop below is bounded by maxAttempts, so keying this test purely + // off the constant under test makes it TAUTOLOGICAL: a regression of + // MAX_CATCHUP_CAP_GAP_ATTEMPTS to 1 -- which is precisely the pre-fix bug this + // test names, a single cap gap killing the sender -- would run the loop ZERO + // times, the "exhausting" attempt would become the FIRST attempt, and the test + // would still pass green. Requiring > 1 makes that regression fail here, and it + // also guarantees the loop runs at least once, so the first cap gap is genuinely + // asserted retriable rather than vacuously skipped. + assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap, else a single " + + "transient failover to a smaller-cap node kills the sender " + + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']', + maxAttempts > 1); // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol. CatchUpCapturingClient client = new CatchUpCapturingClient(160); try (CursorSendEngine engine = newEngine()) { @@ -334,6 +347,13 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); maxField.setAccessible(true); int maxAttempts = maxField.getInt(null); + // Same anti-tautology pin as testCatchUpCapGapRetriesUntilBudgetThenLatches. + // With maxAttempts == 1 the accrual loop below would run ZERO times and the + // "budget accrued to max-1" precondition would degenerate to 0 == 0, so the + // reset-to-0 assertion that is the whole point of this test would prove nothing. + assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap " + + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']', + maxAttempts > 1); CatchUpCapturingClient client = new CatchUpCapturingClient(160); // too small for a 200-char symbol try (CursorSendEngine engine = newEngine()) { CursorWebSocketSendLoop loop = newLoop(engine, client); From 017652fae06d4ab17b568682f2955f575590b34d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:05:53 +0100 Subject: [PATCH 55/78] Keep the producer alive through a torn symbol dictionary Four fixes to the QWP delta symbol-dictionary work, all on the store-and-forward recovery and failover paths. Set an unreplayable slot aside instead of bricking the sender. A host crash can tear the unsynced .symbol-dict relative to the frames it serves, and the resulting frames genuinely cannot be replayed. Detecting that is right; throwing out of build() was not. senderId defaults to a stable name, a not-fully-drained slot is retained on close, and so every subsequent build() re-recovered the same slot and threw again -- forever, until an operator removed the directory by hand. The application could not construct a Sender at all, so it could not even buffer new rows: one already-lost batch became an unbounded outage of everything after it. build() now renames the slot aside, marks it .failed for the orphan drainer, and continues on a fresh empty slot. The bytes are kept for resend; not one of its frames ever reaches the wire. Never recreate a dictionary over an existing file. open() fell through to openFresh() -- which is O_TRUNC -- on any read or parse failure, so a single transient EIO, or a rollback to a client that does not know a newer version byte, destroyed the only copy of load-bearing state and made every surviving delta frame permanently unreplayable. It now degrades to null, leaving the file intact: the sender falls back to full self-sufficient frames and a later attempt can still recover the slot in full. Give the catch-up cap-gap terminal a wall-clock dwell. The settle budget was attempt-based, and 16 attempts at the capped reconnect backoff burn through in about two minutes -- less than an ordinary rolling restart of the larger-cap node, which is the very transient the budget exists to ride out. Escalation now needs both the strike count and a minimum dwell, anchored at the first cap gap of the episode, defaulting to five minutes and tunable through catchup_cap_gap_min_escalation_window_millis. Checksum the persisted dictionary per chunk, not per entry. Crc32c is a native call, so a per-entry checksum put one JNI transition, one sub-cache-line copy and one redundant varint decode on the producer thread for every new symbol -- a thousand native calls per flush on the one-new-symbol-per-row batch this feature exists to serve. A chunk is exactly the set of symbols one frame introduces, and every frame's deltaStart is a chunk boundary, so per-entry granularity bought no extra recoverable prefix. Recovery also folds its two passes into one. ensureScratch now throws instead of clamping below the requested size, which would have turned a clean out-of-memory into a native-heap overflow on the dictionary write path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../main/java/io/questdb/client/Sender.java | 137 +++- .../qwp/client/QwpWebSocketSender.java | 19 +- .../client/sf/cursor/BackgroundDrainer.java | 17 +- .../client/sf/cursor/CursorSendEngine.java | 32 + .../sf/cursor/CursorWebSocketSendLoop.java | 118 +++- .../client/sf/cursor/PersistedSymbolDict.java | 612 ++++++++++-------- .../io/questdb/client/impl/ConfigSchema.java | 1 + .../qwp/client/DeltaDictCatchUpTest.java | 11 +- ...WebSocketSendLoopCatchUpAlignmentTest.java | 124 +++- .../sf/cursor/PersistedSymbolDictTest.java | 260 +++++--- 10 files changed, 974 insertions(+), 357 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4ec70dbd..447d5fad 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -38,6 +38,8 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -988,6 +990,12 @@ final class LineSenderBuilder { // build() time. 0 or negative is a documented "disable" value, so // a Long.MIN_VALUE sentinel keeps it distinguishable from "unset". private static final long DURABLE_ACK_KEEPALIVE_NOT_SET = Long.MIN_VALUE; + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(LineSenderBuilder.class); + // How many quarantined copies of one slot may pile up under sf_dir before build() + // refuses to set aside another. Each is an unreplayable slot a human still has to + // look at; accumulating them without bound would turn a disk-space problem into a + // second incident. + private static final int MAX_QUARANTINE_SLOT_ATTEMPTS = 64; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // sf-client.md section 4.4: the inbox capacity must accommodate the // distinct error categories in a bursty error stream so that drop-oldest @@ -1003,6 +1011,10 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the + // sender's own slot name, so OrphanScanner sees the quarantined copy and the + // orphan drainer can still deliver it if its frames turn out to be replayable. + private static final String QUARANTINE_SLOT_SUFFIX = ".unreplayable-"; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; @@ -1051,6 +1063,7 @@ final class LineSenderBuilder { private int errorInboxCapacity = PARAMETER_NOT_SET_EXPLICITLY; private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY; private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; + private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; private String httpPath; private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; @@ -1505,6 +1518,10 @@ public Sender build() { long actualPoisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY ? poisonMinEscalationWindowMillis : CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + long actualCatchUpCapGapMinEscalationWindowMillis = + catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY + ? catchUpCapGapMinEscalationWindowMillis + : CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; // sfDir is the parent (group root); the actual slot lives // under sfDir/senderId. This is what the engine sees — the @@ -1543,6 +1560,11 @@ public Sender build() { CursorSendEngine cursorEngine = new CursorSendEngine( slotPath, actualSfMaxBytes, actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + if (cursorEngine.isRecoveredDictionaryIncomplete()) { + cursorEngine = quarantineTornSlot( + cursorEngine, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + } int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; @@ -1578,7 +1600,8 @@ public Sender build() { connectionListener, actualConnectionListenerInboxCapacity, actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis ); } catch (Throwable t) { // connect() failed before ownership of cursorEngine @@ -1720,6 +1743,37 @@ public Sender build() { *

* WebSocket transport only. */ + /** + * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must + * persist before the sender gives up on it and fails. + *

+ * A cap gap means a symbol already accepted by one node is too large to + * re-register on the node the sender just failed over to, because that node + * advertises a smaller maximum batch size. On a homogeneous cluster this cannot + * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering + * the cap below existing data. + *

+ * The sender retries such a gap across reconnects rather than failing on sight, + * because the larger-cap node may simply be away -- a rolling restart is the most + * likely reason a failover happened at all. It gives up only once the gap has BOTH + * recurred many times AND persisted for this long, so an ordinary cluster + * operation cannot bring down a live producer. Raise it for a cluster whose + * rolling restarts take longer than the 5-minute default; set it to {@code 0} to + * fail as soon as the retry count is exhausted. + *

+ * WebSocket transport only. + */ + public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + if (millis < 0) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis must be >= 0: ").put(millis); + } + this.catchUpCapGapMinEscalationWindowMillis = millis; + return this; + } + public LineSenderBuilder closeFlushTimeoutMillis(long timeoutMillis) { if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("close_flush_timeout_millis is only supported for WebSocket transport"); @@ -2916,6 +2970,77 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na } } + /** + * Sets a recovered slot whose symbol dictionary cannot cover its surviving frames + * aside, and returns a fresh engine on an empty slot so the producer can keep + * producing. + *

+ * Such a slot is unreplayable BY THIS PRODUCER: its frames reference symbol ids the + * recovered dictionary lost (a host/power crash tore the unsynced side-file), so a + * producer seeded from the short dictionary would hand those ids to different + * symbols and silently misattribute values. Detecting that is correct and + * load-bearing -- but simply THROWING is not a safe response. {@code senderId} + * defaults to a stable name, so a restarted process re-adopts the same slot; the + * engine's close retains a slot that is not fully drained; and so every subsequent + * {@code build()} would re-recover the same slot and throw again -- forever, until + * an operator deleted the directory by hand. The application could not construct a + * Sender at all, and so could not even BUFFER new rows. That trades a bounded, + * already-lost batch for an unbounded outage of everything after it, which inverts + * the one guarantee store-and-forward exists to give. + *

+ * So: rename the slot aside instead. The bytes are preserved for forensics and for + * the orphan drainer, which reaches the same verdict independently (its send loop's + * replay guard fires and it marks the slot {@code .failed}) -- and which, on a slot + * that turns out to be drainable after all (frames written in full-dictionary + * fallback mode are self-sufficient), simply drains it. The new name is NOT the + * sender's own slot name, so {@code OrphanScanner} will consider it. The producer, + * meanwhile, starts on a clean empty slot and never notices. + *

+ * If the rename fails (a Windows share lock, a read-only mount) there is no way to + * free the slot name without destroying data, so fall back to the old behaviour and + * throw -- loudly, and never silently dropping bytes. + */ + private static CursorSendEngine quarantineTornSlot( + CursorSendEngine torn, String sfDir, String senderId, String slotPath, + long sfMaxBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos + ) { + long recoveredMaxSymbolId = torn.recoveredMaxSymbolId(); + PersistedSymbolDict dict = torn.getPersistedSymbolDict(); + int coverage = dict != null ? dict.size() : 0; + String detail = "recovered store-and-forward symbol dictionary cannot cover the surviving " + + "frames (likely a host crash tore its unsynced tail): frames reference symbol id " + + recoveredMaxSymbolId + " but the recovered dictionary holds only " + coverage + + " id(s)"; + // Release the slot lock and the dictionary fd before renaming. The slot is not + // fully drained, so close() retains every file. + torn.close(); + + String quarantinePath = null; + for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { + String candidate = sfDir + "/" + senderId + QUARANTINE_SLOT_SUFFIX + i; + if (!Files.exists(candidate)) { + quarantinePath = candidate; + break; + } + } + if (quarantinePath == null || Files.rename(slotPath, quarantinePath) != 0) { + throw new LineSenderException( + detail + "; the affected data must be resent. The slot could not be set aside " + + "automatically (" + (quarantinePath == null + ? "too many quarantined slots already under " + sfDir + : "rename to " + quarantinePath + " failed") + + "), so this sender cannot start until " + + slotPath + " is moved or removed by hand"); + } + // Mark the quarantined copy so the orphan drainer treats it as a + // human-in-the-loop slot rather than silently retrying it forever. + OrphanScanner.markFailed(quarantinePath, detail); + LOG.error("{} -- the slot has been set aside at {} and the affected data must be resent; " + + "this sender continues on a fresh, empty slot at {}", + detail, quarantinePath, slotPath); + return new CursorSendEngine(slotPath, sfMaxBytes, sfMaxTotalBytes, sfAppendDeadlineNanos); + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); @@ -3413,6 +3538,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "poison_min_escalation_window_millis"); poisonMinEscalationWindowMillis(parseLongValue(sink, "poison_min_escalation_window_millis")); + } else if (Chars.equals("catchup_cap_gap_min_escalation_window_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "catchup_cap_gap_min_escalation_window_millis"); + catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catchup_cap_gap_min_escalation_window_millis")); } else if (Chars.equals("initial_connect_retry", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport"); @@ -3686,6 +3817,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("poison_min_escalation_window_millis")) { poisonMinEscalationWindowMillis(wsLong(view, v, "poison_min_escalation_window_millis")); } + if (view.has("catchup_cap_gap_min_escalation_window_millis")) { + catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catchup_cap_gap_min_escalation_window_millis")); + } if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } @@ -3862,6 +3996,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("max_background_drainers", maxBackgroundDrainers); m.put("max_frame_rejections", maxFrameRejections); m.put("poison_min_escalation_window_millis", poisonMinEscalationWindowMillis); + m.put("catchup_cap_gap_min_escalation_window_millis", catchUpCapGapMinEscalationWindowMillis); m.put("error_inbox_capacity", errorInboxCapacity); m.put("connection_listener_inbox_capacity", connectionListenerInboxCapacity); m.put("token", httpToken); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bc08948e..fb9a7f52 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -313,6 +313,12 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before the + // send loop latches a terminal (connect-string key + // catchup_cap_gap_min_escalation_window_millis). See + // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. + private long catchUpCapGapMinEscalationWindowMillis = + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; private long reconnectInitialBackoffMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS; private long reconnectMaxBackoffMillis = @@ -691,7 +697,8 @@ public static QwpWebSocketSender connect( durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, connectionListenerInboxCapacity, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -722,7 +729,8 @@ public static QwpWebSocketSender connect( SenderConnectionListener connectionListener, int connectionListenerInboxCapacity, int maxFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -740,6 +748,7 @@ public static QwpWebSocketSender connect( sender.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; sender.maxFrameRejections = maxFrameRejections; sender.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + sender.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; @@ -2414,7 +2423,8 @@ public synchronized void startOrphanDrainers( requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); ref[0] = drainer; drainerPool.submit(drainer); } @@ -3342,7 +3352,8 @@ private void ensureConnected() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index d79080d4..a88429ce 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -132,6 +132,11 @@ public final class BackgroundDrainer implements Runnable { // Minimum wall-clock dwell before poison escalation, forwarded to every // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this + // drainer's send loop latches a terminal. Same knob the foreground sender uses; the + // orphan drainer honours it too so a rolling restart cannot quarantine an otherwise + // drainable slot on a strike count alone. + private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( String slotPath, @@ -149,7 +154,8 @@ public BackgroundDrainer( reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -169,7 +175,8 @@ public BackgroundDrainer( boolean requestDurableAck, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; @@ -182,6 +189,7 @@ public BackgroundDrainer( this.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + this.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; } /** @@ -194,7 +202,7 @@ public BackgroundDrainer( @TestOnly public BackgroundDrainer() { this(null, 0L, 0L, null, 0L, 0L, 0L, false, 0L, - CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L); + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L, 0L); } /** @@ -575,7 +583,8 @@ public void run() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); loop.start(); while (!stopRequestedOrInterrupted()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 91035a0e..a2e20d35 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -744,6 +744,38 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } + /** + * Whether this RECOVERED slot's surviving frames reference symbol ids the + * recovered dictionary cannot cover -- i.e. the slot cannot be drained by a + * producer that shares its id space, and a producer seeded from the short + * dictionary would reuse ids those frames already define. + *

+ * True in two situations, both of which mean the same thing to the caller: + *

    + *
  • the dictionary opened but is SHORT of the frames (a host/power crash + * tore its unsynced tail -- see {@code PersistedSymbolDict}'s durability + * note); or
    • + *
  • the dictionary did not open at all ({@code persistedSymbolDict == null} + * -- an unreadable or torn side-file, fd exhaustion, a read-only remount) + * while the surviving frames are DELTA frames that need one. Coverage is + * then zero, so any frame referencing an id at all is unreplayable.
    • + *
+ * Always false for a fresh (non-recovered) slot: {@code recoveredMaxSymbolId} is + * -1 there, and -1 is below every coverage. + *

+ * The caller must NOT drain such a slot with a live producer attached. The + * foreground sender quarantines it and starts on a fresh slot + * ({@code Sender.build()}); the orphan drainer reaches the same verdict + * independently via the send loop's replay guard and marks the slot failed. + * Either way the recorded bytes stay on disk and the affected data must be + * resent -- but the producer keeps producing, which is the whole point of + * store-and-forward. + */ + public boolean isRecoveredDictionaryIncomplete() { + long coverage = persistedSymbolDict != null ? persistedSymbolDict.size() : 0; + return recoveredMaxSymbolId >= coverage; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 93b0b137..8e34011d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -132,6 +132,33 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code LineSenderBuilder.poisonMinEscalationWindowMillis(long)}. */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; + /** + * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must + * persist before the loop latches a terminal, even once + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as + * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a + * strike count measures "how many times did we look", not "how long has this been + * true", and only the latter distinguishes a permanent cluster capability gap from + * a node that is briefly away. + *

+ * It matters here because the cap gap's trigger IS a failover. With the reconnect + * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap + * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling + * restart of the larger-cap node. A count-only budget would therefore hard-fail a + * live store-and-forward producer during a routine cluster operation, which is + * exactly the transient the budget exists to ride out. Requiring the dwell as well + * means the terminal needs BOTH "we have looked many times" AND "it has stayed true + * for a long time"; a node that returns inside the window ends the episode and the + * producer never notices. + *

+ * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, + * this codebase's existing notion of how long a transient outage may plausibly + * last. Configurable per sender via the + * {@code catchup_cap_gap_min_escalation_window_millis} connect-string key or + * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} + * restores count-only escalation. + */ + public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts // -- catch-ups that reached a fresh server and found a single dictionary entry @@ -185,6 +212,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); + // Pre-converted to nanos. Zero disables the dwell entirely (count-only escalation + // at MAX_CATCHUP_CAP_GAP_ATTEMPTS) -- the raw-constructor default; the user-facing + // 5-minute default is applied at the config layer, mirroring + // poisonMinEscalationWindowNanos. + private final long catchUpCapGapMinEscalationWindowNanos; private final CursorSendEngine engine; private final long parkNanos; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only @@ -267,6 +299,16 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // nor resets it. NOT reset per connection -- it measures the cap-gap episode // across reconnects so a persistent gap eventually latches. I/O-thread-only. private int catchUpCapGapAttempts; + // System.nanoTime() of the FIRST cap gap of the current episode, or -1 when no + // episode is open. Anchors the escalation dwell. Reset together with + // catchUpCapGapAttempts on a successful catch-up, so a node that accepts the + // dictionary ends the episode outright. Anchoring at the FIRST gap (not at loop + // entry, and not refreshed per attempt) is what keeps a TRANSIENT from burning the + // terminal budget: a transient never reaches the catch-up, so it neither increments + // the counter nor moves the anchor -- it can only lengthen the wall clock, which + // alone can never latch the terminal because the strike count still has to be + // satisfied. I/O-thread-only, like catchUpCapGapAttempts. + private long catchUpCapGapFirstNanos = -1L; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up consumes // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head @@ -534,14 +576,47 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, 0L); } + /** + * Twelve-arg overload — omits the symbol-dict cap-gap escalation dwell, which + * defaults to {@code 0} (legacy: escalate as soon as + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue, with no minimum + * wall-clock). Like the poison dwell, the user-facing default + * ({@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS}) is applied at the + * config layer (Sender / QwpWebSocketSender), so this raw constructor stays + * unopinionated for tests and internal callers that want deterministic strike-count + * escalation. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, 0L); + } + /** * Master constructor — also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the - * loop escalates to a typed terminal. Must be {@code >= 1}. The final - * argument is the minimum wall-clock dwell (millis) the suspect must stay - * poisoned before escalation ({@code poison_min_escalation_window_millis}; - * {@code >= 0}, where 0 = legacy immediate escalation at the threshold). + * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum + * wall-clock dwell (millis) the suspect must stay poisoned before escalation + * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = legacy + * immediate escalation at the threshold). + *

+ * The final argument is the analogous dwell for the symbol-dict catch-up cap gap + * ({@code catchup_cap_gap_min_escalation_window_millis}; {@code >= 0}, where 0 = + * escalate as soon as {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue). See + * {@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS} for why a strike + * count alone is not a safe escalation signal on a live producer. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -552,7 +627,8 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis) { + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -561,6 +637,13 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException( "poisonMinEscalationWindowMillis must be >= 0: " + poisonMinEscalationWindowMillis); } + if (catchUpCapGapMinEscalationWindowMillis < 0) { + throw new IllegalArgumentException( + "catchUpCapGapMinEscalationWindowMillis must be >= 0: " + + catchUpCapGapMinEscalationWindowMillis); + } + this.catchUpCapGapMinEscalationWindowNanos = + catchUpCapGapMinEscalationWindowMillis * 1_000_000L; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -2314,12 +2397,27 @@ private int sendDictCatchUp() { // recordFatal, NOT fail() -- failing from inside the catch-up would // re-enter connectLoop (see CatchUpSendException); the data must be // resent after the cap is raised. + // Escalation needs BOTH the strike count AND a minimum wall-clock dwell + // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count + // alone measures how often we looked, not how long the gap has held -- + // and 16 attempts at the capped backoff take only ~2 minutes, less than + // an ordinary rolling restart of the larger-cap node. Escalating on the + // count alone would therefore hard-fail a live store-and-forward + // producer during a routine cluster operation. + long nowNanos = System.nanoTime(); + if (catchUpCapGapFirstNanos < 0) { + catchUpCapGapFirstNanos = nowNanos; + } catchUpCapGapAttempts++; - boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS; + long episodeNanos = nowNanos - catchUpCapGapFirstNanos; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS + && episodeNanos >= catchUpCapGapMinEscalationWindowNanos; LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" - + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + ']' + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + + ", episodeMillis=" + (episodeNanos / 1_000_000L) + + '/' + (catchUpCapGapMinEscalationWindowNanos / 1_000_000L) + ']' + (exhausted ? "; the data must be resent after the cap is raised" : "; retrying -- a larger-cap node may return")); @@ -2345,8 +2443,12 @@ private int sendDictCatchUp() { framesSent++; } // The whole dictionary re-registered without a cap gap: this node accepts - // every entry, so the cap-gap episode (if any) is over -- reset the budget. + // every entry, so the cap-gap episode (if any) is over -- reset BOTH the strike + // count and the wall-clock anchor. Resetting only the count would leave a stale + // anchor from an old episode, so the very first strike of a LATER episode would + // already satisfy the dwell. catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; return framesSent; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 83371201..411346d0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -50,20 +50,37 @@ * {@link AckWatermark} -- a discardable optimization protected by a * {@code max()} clamp -- this file is load-bearing: a surviving frame * that references an id missing from it is unrecoverable. It is therefore held - * to a stronger durability contract. + * to a stronger durability contract, and {@link #open} never destroys it (see + * "Never recreate over an existing file" below). *

* Layout (little-endian): * 

  *   offset 0: u32 magic = 'SYD1'
- *   offset 4: u8  version = 2
+ *   offset 4: u8  version = 3
  *   offset 5: 3 bytes reserved (zero)
- *   offset 8: entries, each [len: varint][utf8 bytes][crc32c: u32], in ascending global-id order
+ *   offset 8: chunks, each
+ *             [entryCount: varint][entryBytes: varint][entries][crc32c: u32]
+ *             where entries = [len: varint][utf8] repeated entryCount times,
+ *             occupying exactly entryBytes bytes, and the CRC-32C covers the
+ *             two header varints AND the entry region.
  *
- * Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned - * sequentially from 0), so no id needs to be stored. Each entry carries a - * CRC-32C over its {@code [len][utf8]} bytes (the same checksum the SF segment - * frames use), so a torn or stale entry is detected on recovery instead of - * being silently mis-parsed. + * A chunk is one append -- i.e. exactly the set of symbols one frame + * introduces, since the producer persists a frame's new symbols in a single call + * before publishing it. Symbol id {@code i} is the {@code i}-th entry across all + * chunks (ids are dense and assigned sequentially from 0), so no id is stored. + *

+ * Why the checksum is per chunk, not per entry. The only consumer of the + * recovered prefix is the send loop's replay guard, which compares a surviving + * frame's {@code deltaStart} against the recovered dictionary size -- and every + * {@code deltaStart} is a chunk boundary, because chunks and frame deltas are + * written one-for-one. A tear inside a chunk therefore invalidates exactly the + * frames a per-entry checksum would have invalidated anyway: per-entry + * granularity buys no extra recoverable prefix. It costs a great deal, though. + * {@link Crc32c#update} is a native call, so checksumming per entry put one JNI + * transition -- plus one sub-cache-line copy and one redundant varint decode -- + * on the producer thread for every new symbol. On the high-cardinality batch this + * feature exists to serve (one new symbol per row), that is a thousand native + * calls per flush where the chunk needs one. *

* Durability / write-ahead ordering: the producer appends the symbols a * frame introduces BEFORE that frame is published to the ring, but does NOT @@ -76,14 +93,12 @@ * exactly as the segment frames themselves may be lost on a host crash. Two * layers keep a host-crash tear from silently corrupting data: *

    - *
  • The per-entry CRC-32C: {@link #open} verifies every entry and stops at + *
  • The per-chunk CRC-32C: {@link #open} verifies every chunk and stops at * the first one whose checksum fails, so an interior page lost out of - * order (reading back as zeroes) or a stale entry left past the end by a + * order (reading back as zeroes) or a stale chunk left past the end by a * failed truncate is DETECTED and the trusted region ends before it -- - * recovery never mis-parses a corrupt entry as a real symbol nor shifts - * the dense id->symbol map. The truncate that drops a torn/stale tail is - * now failure-checked (see {@link #open}): a file that cannot be trimmed - * is untrusted and recreated empty rather than left exposing stale bytes.
    • + * recovery never mis-parses a corrupt chunk as real symbols nor shifts the + * dense id->symbol map. *
  • The send loop's replay guard: once recovery trusts only the intact * prefix, a surviving frame whose delta start id exceeds that prefix * fails loudly (the unreplayable data must be resent) rather than sending @@ -93,10 +108,22 @@ * "resend required" instead of a silent symbol misattribution -- the same * CRC-32C protection the segment frames carry. A tear that happened to leave a * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32 - * collision per corrupted entry, no weaker than the frames' own checksum. + * collision per corrupted chunk, no weaker than the frames' own checksum. + *

    + * A torn trailing chunk from a crash mid-append is self-healing: {@link #open} + * stops parsing at the first incomplete chunk and the next append overwrites it. *

    - * A torn trailing entry from a crash mid-append is self-healing: {@link #open} - * stops parsing at the first incomplete entry and the next append overwrites it. + * Never recreate over an existing file. {@link #open} -- the RECOVERY + * entry point -- returns {@code null} when an existing file cannot be read or + * parsed, and NEVER falls back to recreating it empty. Recreating would mean + * {@code O_TRUNC} over the only copy of load-bearing state, so a single transient + * read error (an EIO on a flaky disk, a short read) would permanently destroy the + * dictionary the surviving delta frames reference -- turning a recoverable outage + * into unrecoverable data. A {@code null} instead degrades the sender to full + * self-sufficient frames and leaves every byte on disk, so a later attempt, once + * the transient clears, can still recover the slot in full. Only + * {@link #openClean} -- the FRESH-slot path, where discarding is the whole point + * -- truncates. *

    * Lifecycle: single-writer (the producer / user thread) for appends. Read * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The @@ -115,10 +142,25 @@ public final class PersistedSymbolDict implements QuietCloseable { * OrphanScanner, trim) skip it automatically. */ public static final String FILE_NAME = ".symbol-dict"; - static final int CRC_SIZE = 4; // u32 CRC-32C trailing every entry + static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; - static final byte VERSION = 2; // v2 appended the per-entry CRC-32C + /** + * Upper bound on a chunk's two header varints ({@code entryCount} and + * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The + * encoders reserve this much in front of the entry region so the header can + * be back-filled once the region's exact size is known, keeping header, + * entries and CRC one contiguous run. + */ + static final int MAX_CHUNK_HEADER_SIZE = 10; + /** + * Ceiling for the append scratch buffer, mirroring + * {@code CursorWebSocketSendLoop.MAX_SENT_DICT_BYTES}: the capacity math is + * int-typed, so a larger buffer cannot be addressed. Exceeding it throws -- + * {@link #ensureScratch} never silently under-allocates. + */ + static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8; + static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files); @@ -127,14 +169,14 @@ public final class PersistedSymbolDict implements QuietCloseable { private final FilesFacade ff; private long appendOffset; private boolean closed; - // In-memory copy of the entry region [len][utf8]... exactly as on disk, - // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. READ (not consumed) to - // seed the producer's id map (readLoadedSymbols) and to seed the send loop's - // catch-up mirror, which COPIES it. This file retains ownership for the engine's - // lifetime -- the orphan drainer builds a fresh send loop per wire session - // against the same engine, and each must re-seed its mirror -- and frees this - // buffer in close(). + // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and + // CRCs stripped -- populated only when open() recovered existing chunks + // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not + // consumed) to seed the producer's id map (readLoadedSymbols) and to seed the + // send loop's catch-up mirror, which COPIES it. This file retains ownership for + // the engine's lifetime -- the orphan drainer builds a fresh send loop per wire + // session against the same engine, and each must re-seed its mirror -- and frees + // this buffer in close(). private long loadedEntriesAddr; private int loadedEntriesLen; private long scratchAddr; @@ -151,12 +193,16 @@ private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, } /** - * Opens (creating if absent) the dictionary file in {@code slotDir}. An - * existing file is parsed and its complete entries are loaded into memory - * (see {@link #loadedEntriesAddr()}); a missing or invalid file is (re)created - * with a fresh header. Returns {@code null} on any I/O failure -- the caller - * then falls back to full-dictionary (self-sufficient) frames for this slot, - * so a broken side-file degrades gracefully rather than aborting the sender. + * Opens the dictionary file in {@code slotDir} for RECOVERY, creating it only + * when it does not already exist. An existing file is parsed and its complete, + * CRC-valid chunks are loaded into memory (see {@link #loadedEntriesAddr()}). + *

    + * Returns {@code null} on any I/O or parse failure -- including an existing file + * that cannot be read, carries an unknown version, or fails its checksums. The + * caller then falls back to full-dictionary (self-sufficient) frames for this + * slot, so a broken side-file degrades gracefully rather than aborting the + * sender. Crucially, a {@code null} return NEVER destroys the file: see the + * class-level "Never recreate over an existing file" note. */ public static PersistedSymbolDict open(String slotDir) { return open(FilesFacade.INSTANCE, slotDir); @@ -170,30 +216,27 @@ public static PersistedSymbolDict open(String slotDir) { public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; - // A dictionary that grew to or past Integer.MAX_VALUE cannot be reopened: - // openExisting reads it into ONE int-sized native buffer. PAST 2GiB the - // (int) cast is either negative (malloc rejects it), exactly zero (getInt - // then reads 4 bytes past a zero-size allocation), or a small positive prefix - // (whose validLen < len branch would then DESTRUCTIVELY truncate the multi-GB - // file); AT exactly Integer.MAX_VALUE the cast is exact but the ~2GB malloc is - // doomed to OutOfMemoryError. The >= guard short-circuits every case to a - // clean re-create instead of the doomed allocation -- fail-clean, exactly like - // every other unreadable-file case here, so the sender falls back to full - // self-sufficient frames. Reaching this needs ~100M+ distinct symbols on one - // slot (far past realistic symbol cardinality); the guard keeps the read/write - // size boundary safe anyway. - if (existing >= Integer.MAX_VALUE) { - LOG.warn("symbol dict {} too large ({} bytes) to reopen; recreating empty", filePath, existing); - return openFresh(ff, filePath); - } if (existing >= HEADER_SIZE) { - PersistedSymbolDict d = openExisting(ff, filePath, existing); - if (d != null) { - return d; + // A dictionary at or past Integer.MAX_VALUE cannot be read back: + // openExisting reads it into ONE int-sized native buffer, and the (int) + // cast is either negative (malloc rejects it) or a small positive prefix + // (whose parse would then trim the real multi-GB file). Degrade to full + // self-sufficient frames and leave the file alone. Reaching this needs + // ~100M+ distinct symbols on one slot, far past realistic cardinality; + // the guard keeps the read/write size boundary safe anyway. + if (existing >= Integer.MAX_VALUE) { + LOG.warn("symbol dict {} too large ({} bytes) to reopen; " + + "falling back to full-dictionary frames (file left intact)", filePath, existing); + return null; } - // Fall through to a clean re-create: a header/parse failure on an - // existing file means it cannot be trusted for delta replay. + // NEVER recreate over an existing file on the recovery path: openFresh + // truncates, and these bytes are the only copy of state the surviving + // delta frames reference. A null degrades this slot to full + // self-sufficient frames and preserves the file for a later attempt. + return openExisting(ff, filePath, existing); } + // Absent, or a sub-header stub left by a crash inside openFresh: no + // load-bearing content to lose, so create it. return openFresh(ff, filePath); } @@ -204,14 +247,14 @@ public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { * dictionary left by a prior lifecycle -- a fully-drained slot whose * best-effort delete failed, or a crash in the close window -- must NOT be * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for - * recovery/orphan-drain replay, this truncates it: the fresh-start producer is - * not seeded from the dictionary, so trusting a survivor would leave the - * producer's ids diverged from the dictionary the send loop replays and - * silently misattribute symbols on the next reconnect. Truncating (rather than - * relying on an unlink succeeding first) closes the gap even when the delete is - * refused -- e.g. a Windows share lock. Returns {@code null} on I/O failure, so - * the caller falls back to full self-sufficient frames exactly as {@link #open} - * does. + * recovery/orphan-drain replay and never destroys it, this truncates: the + * fresh-start producer is not seeded from the dictionary, so trusting a survivor + * would leave the producer's ids diverged from the dictionary the send loop + * replays and silently misattribute symbols on the next reconnect. Truncating + * (rather than relying on an unlink succeeding first) closes the gap even when + * the delete is refused -- e.g. a Windows share lock. Returns {@code null} on + * I/O failure, so the caller falls back to full self-sufficient frames exactly + * as {@link #open} does. */ public static PersistedSymbolDict openClean(String slotDir) { return openClean(FilesFacade.INSTANCE, slotDir); @@ -244,28 +287,33 @@ public static void removeOrphan(FilesFacade ff, String slotDir) { /** * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the - * symbol-dict delta section the frame encoder just wrote -- to the on-disk - * dictionary, computing and appending a per-entry CRC-32C as it copies so the - * producer does not re-encode the symbols. The on-disk layout is - * {@code [len varint][utf8][crc32c]} per entry (see the class layout note); the - * {@code addr}/{@code len} bytes carry no CRC, so this walks the {@code count} - * entries to insert one. Advances {@code size} by {@code count}. Same - * durability/idempotency contract as {@link #appendSymbols}: no fsync, and a - * short write throws WITHOUT advancing {@code size}/{@code appendOffset}, so a - * retry keyed off {@link #size()} re-persists the same range at the same - * offset. No-op when the range is empty or the dictionary is closed. + * symbol-dict delta section the frame encoder just wrote -- as ONE chunk. + *

    + * The consistency walk below decodes each entry's length varint, but the bytes + * themselves are copied in a SINGLE {@code copyMemory} and checksummed by a + * SINGLE {@link Crc32c#update} covering the whole chunk. A per-entry checksum + * would put one JNI transition, one sub-cache-line copy and one redundant varint + * decode on the producer thread per new symbol; the chunk needs one of each. + *

    + * Advances {@code size} by {@code count}. Same durability/idempotency contract + * as {@link #appendSymbols}: no fsync, and a short write throws WITHOUT + * advancing {@code size}/{@code appendOffset}, so a retry keyed off + * {@link #size()} re-persists the same range at the same offset. No-op when the + * range is empty or the dictionary is closed. */ public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } - int outLen = len + count * CRC_SIZE; - ensureScratch(outLen); + // Validate the (addr, len, count) triple BEFORE writing anything: an + // inconsistent triple would record a chunk whose stored entryCount disagreed + // with the entries it holds, shifting the dense id->symbol map on recovery. + // The sole caller derives count and len from one beginMessage, so this cannot + // fire today -- but the file this guards is the one the "resend required" + // contract rests on, so it stays. long src = addr; long srcLimit = addr + len; - long dst = scratchAddr; for (int i = 0; i < count; i++) { - long entryStart = src; long symLen = 0; int shift = 0; while (src < srcLimit) { @@ -277,51 +325,37 @@ public synchronized void appendRawEntries(long addr, int len, int count) { shift += 7; if (shift > 35) { // A canonical entry-length varint is <= 5 bytes; a longer - // continuation run is corrupt. The downstream entryEnd > srcLimit - // check then rejects it. Matches decodeVarint / readVarintAt. + // continuation run is corrupt. The bound check below rejects it. break; } } - long entryEnd = src + symLen; // src is just past the len varint - if (entryEnd > srcLimit) { + src += symLen; // src was just past the len varint + if (src > srcLimit) { throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME + " [entry=" + i + ", count=" + count + ']'); } - int wireSpan = (int) (entryEnd - entryStart); // [len][utf8] - Unsafe.getUnsafe().copyMemory(entryStart, dst, wireSpan); - Unsafe.getUnsafe().putInt(dst + wireSpan, Crc32c.update(Crc32c.INIT, entryStart, wireSpan)); - dst += wireSpan + CRC_SIZE; - src = entryEnd; } if (src != srcLimit) { - // The count entries did not consume exactly len bytes -- a caller passed an - // inconsistent (addr, len, count) triple. Writing outLen would flush an - // uninitialised scratch tail and mis-advance size, so fail loudly. The sole - // caller derives count and len from one beginMessage, so this cannot fire - // today. throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " + FILE_NAME + " [count=" + count + ", len=" + len + ", consumed=" + (int) (src - addr) + ']'); } - long written = ff.write(fd, scratchAddr, outLen, appendOffset); - if (written != outLen) { - throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + outLen + ", actual=" + written + ']'); - } - appendOffset += outLen; - size += count; + int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len); + ensureScratch((long) hdrLen + len + CRC_SIZE); + long p = NativeBufferWriter.writeVarint(scratchAddr, count); + NativeBufferWriter.writeVarint(p, len); + Unsafe.getUnsafe().copyMemory(addr, scratchAddr + hdrLen, len); + flushChunk(scratchAddr, hdrLen, len, count); } /** - * Appends one symbol, extending the on-disk dictionary. The caller appends a - * frame's new symbols BEFORE publishing that frame, so the write ordering - * (dictionary entry before referencing frame) holds; no fsync is performed - * (see the class-level durability note). Assigns the next dense id implicitly - * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC - * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on - * recovery. + * Appends one symbol as a single-entry chunk, extending the on-disk dictionary. + * The caller appends a frame's new symbols BEFORE publishing that frame, so the + * write ordering (dictionary entry before referencing frame) holds; no fsync is + * performed (see the class-level durability note). Assigns the next dense id + * implicitly (the entry's position). *

    - * Test-only: production persists a frame's whole new-symbol range in one write + * Test-only: production persists a frame's whole new-symbol range in one chunk * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to * build a dictionary one entry at a time. */ @@ -331,65 +365,49 @@ public synchronized void appendSymbol(CharSequence symbol) { return; } int utf8Len = Utf8s.utf8Bytes(symbol); - int varLen = NativeBufferWriter.varintSize(utf8Len); - int wireLen = varLen + utf8Len; // [len][utf8] - int recLen = wireLen + CRC_SIZE; // + trailing crc - ensureScratch(recLen); - long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE); + long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE; + long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } - Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen)); - long written = ff.write(fd, scratchAddr, recLen, appendOffset); - if (written != recLen) { - throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + recLen + ", actual=" + written + ']'); - } - appendOffset += recLen; - size++; + writeChunkFromScratch(wireLen, 1); } /** - * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes - * the whole {@code [len varint][utf8][crc32c]...} region into scratch first, - * then issues one positioned write -- versus one {@code pwrite(2)} per symbol - * via {@link #appendSymbol}. That per-symbol syscall count is the hot-path - * cost on a high-cardinality batch (one new symbol per row), which is exactly - * the store-and-forward workload delta encoding targets. Each entry carries a - * trailing CRC-32C over its {@code [len][utf8]} bytes. Callers pass the - * dictionary and the range so the ids resolve to their symbol strings. + * Appends the dense id range {@code [from .. to]} as ONE chunk, in a SINGLE + * write with a SINGLE checksum. Encodes the whole entry region into scratch + * first -- versus one {@code pwrite(2)} and one native CRC call per symbol. + * That per-symbol syscall and JNI count is the hot-path cost on a + * high-cardinality batch (one new symbol per row), which is exactly the + * store-and-forward workload delta encoding targets. Callers pass the dictionary + * and the range so the ids resolve to their symbol strings. *

    - * Same durability and idempotency contract as {@link #appendSymbol}: no - * fsync, and a short write throws WITHOUT advancing {@code size}/{@code - * appendOffset}, so a retry keyed off {@link #size()} re-encodes the same - * range and overwrites at the same offset. No-op when the range is empty or - * the dictionary is closed. + * Same durability and idempotency contract as {@link #appendSymbol}: no fsync, + * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset}, + * so a retry keyed off {@link #size()} re-encodes the same range and overwrites + * at the same offset. No-op when the range is empty or the dictionary is closed. */ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { if (closed || to < from) { return; } - int len = 0; + int count = to - from + 1; + int entriesLen = 0; for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] - ensureScratch(len + wireLen + CRC_SIZE); - long entryStart = scratchAddr + len; + ensureScratch((long) MAX_CHUNK_HEADER_SIZE + entriesLen + wireLen + CRC_SIZE); + long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE + entriesLen; long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } - Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen)); - len += wireLen + CRC_SIZE; + entriesLen += wireLen; } - long written = ff.write(fd, scratchAddr, len, appendOffset); - if (written != len) { - throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + len + ", actual=" + written + ']'); - } - appendOffset += len; - size += to - from + 1; + writeChunkFromScratch(entriesLen, count); } @Override @@ -418,8 +436,9 @@ public synchronized void close() { /** * Base address of the loaded entry region -- the concatenated - * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly - * as a delta section carries them. Zero when nothing was recovered. + * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a + * delta section carries them (chunk headers and CRCs stripped). Zero when + * nothing was recovered. *

    * Construction-phase only. This hands out a raw pointer into native * memory that {@link #close()} frees and nulls, with no closed-guard and no @@ -466,10 +485,10 @@ public ObjList readLoadedSymbols() { } shift += 7; if (shift > 35) { - // Bound the varint like decodeVarint / appendRawEntries / - // readVarintAt: a canonical length is <= 5 bytes. open() already - // CRC-validated these bytes, so this is defensive only; the - // p + len > limit check below then rejects the over-long run. + // Bound the varint like the other decoders: a canonical length is + // <= 5 bytes. open() already CRC-validated these bytes, so this is + // defensive only; the p + len > limit check below rejects the + // over-long run. break; } } @@ -489,47 +508,25 @@ public int size() { return size; } - /** - * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns - * {@code [value, newPos]} or {@code null} if the varint is truncated - * (torn tail). - */ - private static long[] decodeVarint(long buf, int pos, int limit) { - long value = 0; - int shift = 0; - int cur = pos; - while (cur < limit) { - byte b = Unsafe.getUnsafe().getByte(buf + cur); - cur++; - value |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return new long[]{value, cur}; - } - shift += 7; - if (shift > 35) { - return null; // implausible for an entry length; treat as torn - } - } - return null; - } - private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen) { int fd = ff.openRW(filePath); if (fd < 0) { - LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd); + LOG.warn("symbol dict {} could not be opened (rc={}); " + + "falling back to full-dictionary frames (file left intact)", filePath, fd); return null; } long buf = 0L; long entriesAddr = 0L; - int entriesLen = 0; + int entriesCap = 0; try { - int len = (int) fileLen; + int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE) buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); long read = ff.read(fd, buf, len, 0); if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { - LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); + LOG.warn("symbol dict {} unreadable, bad magic or unknown version; " + + "falling back to full-dictionary frames (file left intact)", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // null after free so the catch below cannot double-free if ff.close throws int fdToClose = fd; @@ -537,92 +534,89 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, ff.close(fdToClose); return null; } - // Parse complete, CRC-valid entries after the header; stop at the first - // torn/incomplete OR crc-mismatched entry. The CRC turns an interior - // tear (a lost page reading back as zeroes) or a stale entry left past - // the end by a failed truncate into a clean stop point, so recovery - // trusts only the intact prefix instead of silently mis-parsing a - // corrupt entry and shifting the dense id->symbol map. - int diskPos = HEADER_SIZE; // walks the on-disk [len][utf8][crc] entries + // Parse the chunks after the header, copying each chunk's entry region + // into entriesAddr AS WE VALIDATE IT -- one pass over the file, not two. + // Every chunk sheds its two header varints and its CRC, so the entry + // region is strictly smaller than the file and len - HEADER_SIZE is a safe + // upper bound to allocate up front; we shrink to the exact size below. + // + // Stop at the first torn/incomplete OR crc-mismatched chunk. The CRC turns + // an interior tear (a lost page reading back as zeroes) or a stale chunk + // left past the end by a failed truncate into a clean stop point, so + // recovery trusts only the intact prefix instead of silently mis-parsing a + // corrupt chunk and shifting the dense id->symbol map. + entriesCap = len - HEADER_SIZE; + long dst = 0L; + if (entriesCap > 0) { + entriesAddr = Unsafe.malloc(entriesCap, MemoryTag.NATIVE_DEFAULT); + dst = entriesAddr; + } + Varint v = new Varint(); + int diskPos = HEADER_SIZE; int count = 0; - int wireLen = 0; // running size of the crc-stripped copy while (diskPos < len) { - long[] vr = decodeVarint(buf, diskPos, len); - if (vr == null) { - break; // torn length varint + if (!v.decode(buf, diskPos, len)) { + break; // torn entryCount varint + } + long entryCount = v.value; + if (!v.decode(buf, v.end, len)) { + break; // torn entryBytes varint } - long symLen = vr[0]; - int afterVar = (int) vr[1]; - // [len varint][utf8] then a u32 CRC. symLen stays a long so a - // corrupt multi-gigabyte length cannot wrap an int back under the - // bound check. No fixed per-entry ceiling -- the write path applies - // none, so a legitimately large symbol must recover here. - long wireEnd = (long) afterVar + symLen; // end of [len][utf8] - if (wireEnd + CRC_SIZE > len) { - break; // torn/incomplete trailing entry (its CRC doesn't fit) + long entryBytes = v.value; + int entriesStart = v.end; + // entryCount/entryBytes stay long so a corrupt multi-gigabyte value + // cannot wrap an int back under the bound checks. + long chunkEnd = (long) entriesStart + entryBytes; // end of the entry region + if (chunkEnd + CRC_SIZE > len) { + break; // torn/incomplete trailing chunk (its CRC doesn't fit) } - int wireEndI = (int) wireEnd; - int wireSpan = wireEndI - diskPos; - int crcStored = Unsafe.getUnsafe().getInt(buf + wireEndI); - int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, wireSpan); + int chunkEndI = (int) chunkEnd; + int crcStored = Unsafe.getUnsafe().getInt(buf + chunkEndI); + int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, chunkEndI - diskPos); if (crcCalc != crcStored) { - break; // corrupt/stale entry -- stop before it (fail-clean) + break; // corrupt/stale chunk -- stop before it (fail-clean) } - diskPos = wireEndI + CRC_SIZE; - wireLen += wireSpan; - count++; - } - int diskConsumed = diskPos - HEADER_SIZE; // valid entries incl. their CRCs - // Materialise the trusted entries as WIRE bytes ([len][utf8]..., no - // CRC) so loadedEntries*/readLoadedSymbols and the send-loop catch-up - // mirror stay wire-shaped -- the on-disk CRC is stripped here, once, at - // open. A second no-alloc walk over the already-validated region. - if (wireLen > 0) { - entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT); - // Record the length alongside the malloc so the catch below frees the - // right size (not 0) if this copy walk ever throws. - entriesLen = wireLen; - long dst = entriesAddr; - int p = HEADER_SIZE; - for (int i = 0; i < count; i++) { - int vp = p; - long symLen = 0; - int shift = 0; - while (true) { - byte b = Unsafe.getUnsafe().getByte(buf + vp++); - symLen |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; // corrupt run; these entries were CRC-validated above - } - } - int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC - Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan); - dst += wireSpan; - p += wireSpan + CRC_SIZE; // skip the entry's CRC + // A chunk carries at least one entry, and the ids are int-dense. A + // CRC-valid chunk cannot violate either at realistic cardinality, but + // keep the int narrowing honest rather than wrapping the id space. + if (entryCount <= 0 || (long) count + entryCount > Integer.MAX_VALUE) { + break; } + Unsafe.getUnsafe().copyMemory(buf + entriesStart, dst, entryBytes); + dst += entryBytes; + diskPos = chunkEndI + CRC_SIZE; + count += (int) entryCount; + } + int entriesLen = entriesAddr != 0L ? (int) (dst - entriesAddr) : 0; + int diskConsumed = diskPos - HEADER_SIZE; // valid chunks incl. headers and CRCs + if (entriesAddr != 0L && entriesLen == 0) { + Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT); + entriesAddr = 0L; + entriesCap = 0; + } else if (entriesAddr != 0L && entriesLen < entriesCap) { + // Shrink the upper-bound allocation to what the trusted prefix used. + entriesAddr = Unsafe.realloc(entriesAddr, entriesCap, entriesLen, MemoryTag.NATIVE_DEFAULT); + entriesCap = entriesLen; } Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // Drop any torn/stale trailing bytes so a LATER, shorter append cannot - // leave residue past its own end. Unlike before, the truncate result IS - // checked: a file we cannot trim could still expose stale post-end bytes - // whose (self-consistent) per-entry CRC the parse would accept at a - // shifted position, so a failed truncate makes the file untrusted -- - // open() then recreates it empty (fail-clean) rather than risk a silent - // misattribution. + // leave residue past its own end. The truncate result IS checked: a file + // we cannot trim could still expose stale post-end bytes whose + // (self-consistent) chunk CRC the parse would accept at a shifted + // position, so a failed truncate makes the file untrusted -- return null + // (the sender falls back to full self-sufficient frames) and, per the + // never-destroy contract, leave every byte on disk. long validLen = HEADER_SIZE + diskConsumed; if (validLen < len && !ff.truncate(fd, validLen)) { - LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); + LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); " + + "falling back to full-dictionary frames (file left intact)", filePath); if (entriesAddr != 0L) { - Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT); // Null after freeing (like buf above) so the catch below cannot // double-free entriesAddr if the following ff.close throws. entriesAddr = 0L; - entriesLen = 0; + entriesCap = 0; } int fdToClose = fd; fd = -1; // relinquish before close so the catch cannot double-close if close throws @@ -640,12 +634,16 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, // double-free. Keeps the error path leak-free on any throw between its // malloc and the return. if (entriesAddr != 0L) { - Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT); } if (fd >= 0) { // a branch that already closed fd relinquished it to -1 ff.close(fd); } - LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); + // Pass the throwable as a trailing argument with no matching placeholder so + // slf4j prints the stack trace: this WARN is the only forensic record of why + // a load-bearing dictionary was abandoned. + LOG.warn("symbol dict {} recovery failed; falling back to full-dictionary frames " + + "(file left intact)", filePath, t); return null; } } @@ -669,19 +667,23 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { int fdToClose = fd; fd = -1; // relinquish before close so the catch cannot double-close if close throws ff.close(fdToClose); - ff.remove(filePath); + ff.remove(filePath); // drop the headerless stub rather than litter LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; } } catch (Throwable t) { - // Unreachable today (Files.write is native and returns -1 rather than - // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte - // malloc cannot realistically OOM), but close the fd against a future - // edit so it cannot leak -- mirroring openExisting's error handling. + // Unreachable with FilesFacade.INSTANCE (Files.write is native and returns + // -1 rather than throwing; the Unsafe puts target a valid 8-byte buffer and + // an 8-byte malloc cannot realistically OOM), but the ff seam exists so + // tests CAN inject a throwing facade -- close the fd and drop the stub so + // neither leaks. if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing - ff.close(fd); + int fdToClose = fd; + fd = -1; + ff.close(fdToClose); + ff.remove(filePath); } - LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); + LOG.warn("symbol dict {} creation failed; proceeding without it", filePath, t); return null; } finally { if (hdr != 0L) { @@ -691,19 +693,107 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0); } - private void ensureScratch(int required) { + /** + * Grows the append scratch buffer to hold at least {@code required} bytes. + *

    + * Throws when {@code required} exceeds {@link #MAX_SCRATCH_BYTES} rather than + * clamping to it: a clamp would hand back a buffer SMALLER than the caller asked + * for and return normally, and every caller then writes {@code required} bytes + * into it -- turning a clean out-of-memory into a silent native-heap overflow, on + * the very write path the dictionary's integrity rests on. This is the same + * loud-failure shape {@code CursorWebSocketSendLoop.ensureSentDictCapacity} uses + * on the same bound. Unreachable at any realistic cardinality (it needs a ~2 GiB + * dictionary section in a single frame, itself bounded by the server's batch + * cap), but a size guard on a data-integrity write path must never + * under-allocate. + */ + private void ensureScratch(long required) { if (scratchCap >= required) { return; } + if (required > MAX_SCRATCH_BYTES) { + throw new IllegalStateException("symbol dict scratch buffer exceeds the maximum size to " + + FILE_NAME + " [required=" + required + ", max=" + MAX_SCRATCH_BYTES + ']'); + } // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and - // would make the realloc size negative. required is bounded by one frame's - // entries (the server batch cap), so this never actually caps -- it mirrors the - // long-math growth in CursorWebSocketSendLoop.ensureSentDictCapacity. + // would make the realloc size negative. long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2)); - if (newCap > Integer.MAX_VALUE - 8) { - newCap = Integer.MAX_VALUE - 8; + if (newCap > MAX_SCRATCH_BYTES) { + newCap = MAX_SCRATCH_BYTES; } scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT); scratchCap = (int) newCap; } + + /** + * Checksums {@code [recStart, recStart + hdrLen + entriesLen)} in ONE native + * call, appends the CRC, and issues ONE positioned write. Advances + * {@code size}/{@code appendOffset} only on a complete write, so a short write + * throws and a retry keyed off {@link #size()} re-persists the same range at the + * same offset. + */ + private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) { + int bodyLen = hdrLen + entriesLen; + int recLen = bodyLen + CRC_SIZE; + Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen)); + long written = ff.write(fd, recStart, recLen, appendOffset); + if (written != recLen) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + recLen + ", actual=" + written + ']'); + } + appendOffset += recLen; + size += count; + } + + /** + * Writes one chunk whose entry region is ALREADY encoded in scratch at offset + * {@link #MAX_CHUNK_HEADER_SIZE}. Back-fills the header immediately in front of + * the entries -- the header is at most {@code MAX_CHUNK_HEADER_SIZE} bytes, so it + * always fits the reserve -- leaving header, entries and CRC one contiguous run + * for a single checksum and a single write. + */ + private void writeChunkFromScratch(int entriesLen, int count) { + int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(entriesLen); + long recStart = scratchAddr + MAX_CHUNK_HEADER_SIZE - hdrLen; + long p = NativeBufferWriter.writeVarint(recStart, count); + NativeBufferWriter.writeVarint(p, entriesLen); + flushChunk(recStart, hdrLen, entriesLen, count); + } + + /** + * Zero-allocation LEB128 decoder, one instance per {@link #openExisting} call -- + * not one per chunk. The previous {@code long[]}-returning decoder allocated once + * per ENTRY, so a million-symbol recovery churned a million throwaway arrays in a + * class that is otherwise strictly allocation-free. + */ + private static final class Varint { + int end; + long value; + + /** + * Decodes the varint at {@code buf[pos..limit)}. Returns false -- leaving + * {@code value}/{@code end} undefined -- when the varint is truncated (a torn + * tail) or runs longer than a canonical 5-byte length. + */ + boolean decode(long buf, int pos, int limit) { + long v = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + v |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + value = v; + end = cur; + return true; + } + shift += 7; + if (shift > 35) { + return false; // implausible for a chunk header; treat as torn + } + } + return false; + } + } } diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index a255e638..7097f3cd 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -77,6 +77,7 @@ public final class ConfigSchema { // (one vocabulary, side-owned application). str("max_frame_rejections", Side.INGRESS); str("poison_min_escalation_window_millis", Side.INGRESS); + str("catchup_cap_gap_min_escalation_window_millis", Side.INGRESS); str("max_name_len", Side.INGRESS); str("reconnect_initial_backoff_millis", Side.INGRESS); str("reconnect_max_backoff_millis", Side.INGRESS); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 132f2c06..db7603f2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -238,8 +238,17 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry LineSenderException terminal = null; + // catchup_cap_gap_min_escalation_window_millis=0 restores count-only + // escalation, so the terminal latches as soon as the settle budget's + // strikes are exhausted. The DEFAULT is a 5-minute wall-clock dwell on + // top of the strike count, precisely so a cap gap that lasts only as + // long as a rolling restart cannot hard-fail a live producer -- and a + // test cannot wait that out. Zeroing it here is what makes the terminal + // observable; the dwell itself is covered by + // CursorWebSocketSendLoopCatchUpAlignmentTest. Sender sender = Sender.fromConfig("ws::addr=localhost:" + port - + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;"); + + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50" + + ";catchup_cap_gap_min_escalation_window_millis=0;"); try { sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); sender.flush(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 49411275..707ef5a8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -40,6 +40,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.TimeUnit; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -266,6 +267,112 @@ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { }); } + @Test + public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() throws Exception { + // The strike count alone must NOT latch a terminal: escalation also requires the + // cap-gap episode to have persisted for catchUpCapGapMinEscalationWindowMillis. + // + // This is what keeps a routine rolling restart from killing a live producer. + // MAX_CATCHUP_CAP_GAP_ATTEMPTS strikes accrue in ~2 minutes at the capped + // reconnect backoff -- less than the time the larger-cap node is away -- so a + // count-only budget would hard-fail the sender on the very transient the budget + // exists to ride out. Here we drive far MORE than the budget's strikes inside a + // (deliberately huge) window and assert the sender stays alive. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + // A one-hour dwell the test cannot possibly elapse. + CursorWebSocketSendLoop loop = newLoop(engine, client, 3_600_000L); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + for (int i = 1; i <= maxAttempts + 4; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + // The producer-facing latch must stay clear on EVERY attempt, + // including the ones past the strike budget. + loop.checkError(); + } + assertTrue("the strikes really did exceed the budget", + readInt(loop, "catchUpCapGapAttempts") > maxAttempts); + + // Backdate the episode anchor past the window: the very next cap gap + // now satisfies BOTH conditions and latches. This pins the AND -- if + // escalation ignored the wall clock the loop would already have + // latched above; if it ignored the strike count it could never latch. + Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos"); + anchor.setAccessible(true); + anchor.setLong(loop, System.nanoTime() - TimeUnit.HOURS.toNanos(2)); + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("the escalating cap gap must still raise CatchUpSendException"); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + try { + loop.checkError(); + fail("a cap gap that outlives the escalation window must latch a terminal"); + } catch (LineSenderException terminal) { + assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up") + && terminal.getMessage().contains("must be resent")); + } + } finally { + loop.close(); + } + } + }); + } + + @Test + public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Exception { + // A TRANSIENT catch-up failure (the wire drops mid-catch-up -- a flapping LB, a + // reset) must never increment the cap-gap terminal budget. The budget exists to + // prove a PERSISTENT cluster capability gap; letting a transient feed it means + // enough wire flaps hard-fail a live store-and-forward producer, which is the + // exact failure store-and-forward promises cannot happen. + // + // The production code is correct, but nothing pinned it: the counter is never + // read by the existing transient test, and one transient can never reach a + // 16-strike budget anyway. So drive MORE transients than the whole budget and + // assert the counter never moves and no terminal ever latches. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + // A cap that FITS (no cap gap), but whose sendBinary always throws: every + // failure here is transport-transient, never a capability gap. + CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client, 0L); + try { + seedMirror(loop, "alpha"); + for (int i = 1; i <= maxAttempts + 4; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("a transient send failure must raise CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); // a transient is retriable, forever + assertEquals("a transient must NOT burn the cap-gap terminal budget", + 0, readInt(loop, "catchUpCapGapAttempts")); + assertEquals("a transient must NOT anchor a cap-gap episode", + -1L, readLong(loop, "catchUpCapGapFirstNanos")); + } + } finally { + loop.close(); + } + } + }); + } + @Test public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // M1: an entry too large for the fresh server's cap during catch-up (a @@ -539,12 +646,27 @@ private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loo } private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { + return newLoop(engine, client, 0L); + } + + /** + * As {@link #newLoop(CursorSendEngine, WebSocketClient)} but with an explicit + * cap-gap escalation dwell. {@code 0} (the default here) means count-only + * escalation, which is what the raw constructor gives and what the strike-count + * tests want; the user-facing 5-minute default is applied at the config layer. + */ + private CursorWebSocketSendLoop newLoop( + CursorSendEngine engine, WebSocketClient client, long capGapWindowMillis + ) { return new CursorWebSocketSendLoop( client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, () -> { throw new UnsupportedOperationException("test loop is never started"); }, - 5_000L, 100L, 5_000L, false); + 5_000L, 100L, 5_000L, false, + CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, capGapWindowMillis); } private CursorSendEngine newEngine() { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 677e12e2..b1255aac 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -41,6 +41,11 @@ public class PersistedSymbolDictTest { + // On-disk geometry, mirrored from PersistedSymbolDict so the layout-derived + // corruption tests below read as arithmetic rather than magic numbers. + private static final int CRC_SIZE = 4; + private static final int HEADER_SIZE = 8; + @Test public void testAppendPersistsAcrossReopen() throws Exception { assertMemoryLeak(() -> { @@ -331,22 +336,88 @@ public void testAppendSymbolsShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry } @Test - public void testBadMagicIsRecreatedEmpty() throws Exception { + public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { + // A batch append must write ONE chunk -- one [entryCount][entryBytes] header and + // ONE trailing CRC-32C for the whole range -- not one checksummed record per + // symbol. + // + // This is the load-bearing property of the v3 layout, and it is a producer-thread + // cost, not a cosmetic one: Crc32c.update is a NATIVE call, so a per-entry + // checksum put one JNI transition (plus one sub-cache-line copy and one redundant + // varint decode) on the flush path for every new symbol. On the high-cardinality + // batch this whole feature exists to serve -- one new symbol per row -- that is a + // thousand native calls per flush where the chunk needs one. + // + // Asserted through the file size, which is exact and cheap: N single-byte ASCII + // symbols cost N * ([len varint] + 1 utf8 byte) of entries, wrapped in one chunk. + // Per-entry CRCs would add 4 * N bytes instead of 4. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { - // A file with the right size but garbage content (bad magic). - Path f = dir.resolve(".symbol-dict"); - Files.write(f, new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); + final int n = 26; // 'a'..'z' -- distinct, and one UTF-8 byte each + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + for (int i = 0; i < n; i++) { + dict.getOrAddSymbol(String.valueOf((char) ('a' + i))); + } + Assert.assertEquals(n, dict.size()); PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(d); try { - Assert.assertEquals("bad-magic file recreated empty", 0, d.size()); - d.appendSymbol("X"); - Assert.assertEquals(1, d.size()); + d.appendSymbols(dict, 0, n - 1); + Assert.assertEquals(n, d.size()); } finally { d.close(); } + + int entryBytes = n * 2; // [len=1 varint][1 utf8 byte] + int chunkHeader = 1 + varintSize(entryBytes); // entryCount=50 fits one byte + long expected = HEADER_SIZE + chunkHeader + entryBytes + CRC_SIZE; + Assert.assertEquals( + "a batch append must cost ONE chunk checksum, not one per symbol", + expected, Files.size(dir.resolve(".symbol-dict"))); + + // And it must still read back symbol-for-symbol. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + Assert.assertEquals(n, re.size()); + ObjList got = re.readLoadedSymbols(); + for (int i = 0; i < n; i++) { + Assert.assertEquals(dict.getSymbol(i), got.getQuick(i)); + } + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + + private static int varintSize(int v) { + int n = 1; + while ((v >>>= 7) != 0) { + n++; + } + return n; + } + + @Test + public void testBadMagicDegradesWithoutDestroyingTheFile() throws Exception { + // An unparseable existing file must degrade to null -- the sender falls back + // to full self-sufficient frames -- and must NOT be recreated. open() used to + // fall through to openFresh(), which is O_TRUNC: a single unreadable byte + // destroyed the only copy of load-bearing state. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + Path f = dir.resolve(".symbol-dict"); + byte[] original = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + Files.write(f, original); + Assert.assertNull("an unparseable existing dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy an existing dictionary", + original, Files.readAllBytes(f)); } finally { rmDir(dir); } @@ -354,36 +425,35 @@ public void testBadMagicIsRecreatedEmpty() throws Exception { } @Test - public void testBadVersionIsRecreatedEmpty() throws Exception { - // A file with correct 'SYD1' magic and a VALID entry but an unknown version - // byte must be recreated empty -- its entries belong to a foreign/future - // format and must not be parsed as v2. Covers the version sub-condition - // specifically (testBadMagicIsRecreatedEmpty covers the magic one). Because - // the seeded "a" is a valid v2 entry, without the version check it would parse - // back (size 1), so this fails. + public void testBadVersionDegradesWithoutDestroyingTheFile() throws Exception { + // A file with correct 'SYD1' magic and a VALID chunk but an unknown version + // byte belongs to a foreign/future format and must not be parsed as v3. + // Covers the version sub-condition specifically (testBadMagicDegrades... covers + // the magic one). + // + // The file must SURVIVE. This is the client-rollback trap: a newer client + // writes v4, ops roll back to this build, and if open() recreated the file the + // v4 dictionary would be gone for good -- rolling forward again could not + // recover it, and every surviving delta frame would be permanently + // unreplayable. Degrading to null instead leaves the bytes for the client that + // does understand them. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { Path f = dir.resolve(".symbol-dict"); - // Build a real v2 dictionary with one valid entry... PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(seed); seed.appendSymbol("a"); seed.close(); - // ...then corrupt ONLY the version byte (offset 4) to an unknown value. + // Corrupt ONLY the version byte (offset 4) to an unknown value. byte[] bytes = Files.readAllBytes(f); bytes[4] = (byte) 99; Files.write(f, bytes); - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - Assert.assertNotNull(d); - try { - Assert.assertEquals("bad-version file recreated empty (entries must not parse)", 0, d.size()); - d.appendSymbol("X"); - Assert.assertEquals(1, d.size()); - } finally { - d.close(); - } + Assert.assertNull("an unknown-version dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy a future-version dictionary", + bytes, Files.readAllBytes(f)); } finally { rmDir(dir); } @@ -446,11 +516,11 @@ public void testEmptySymbolRoundTrips() throws Exception { @Test public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exception { // A host-crash interior tear (a lost page reading back as zeroes) or a - // stale entry left past the end by a failed truncate can change the bytes - // of a NON-trailing entry. Without the per-entry CRC the parse would + // stale chunk left past the end by a failed truncate can change the bytes + // of a NON-trailing chunk. Without the per-chunk CRC the parse would // accept those bytes, shifting the dense id->symbol map and silently // misattributing symbol-column values on replay. With the CRC the corrupt - // entry fails verification and the parse stops there, so recovery trusts + // chunk fails verification and the parse stops there, so recovery trusts // only the intact prefix (fail-clean: the send loop's torn-dict guard then // forces a resend of the rest). assertMemoryLeak(() -> { @@ -468,25 +538,27 @@ public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exce d.close(); } - // Corrupt one byte inside the 3rd entry's UTF-8 (id 2). On-disk - // entry layout is [len varint][utf8][crc32c u32]; a 2-byte ASCII - // symbol is 1 + 2 + 4 = 7 bytes, after the 8-byte header: - // header[0,8) e0[8,15) e1[15,22) e2[22,29) ... - // Offset 23 is "s2"'s first UTF-8 byte; flipping it leaves e2's - // stored CRC stale. + // Corrupt one byte inside the 3rd chunk's entry region (id 2). Each + // appendSymbol writes one single-entry chunk + // ([entryCount][entryBytes][len][utf8][crc32c]), and all five symbols + // are the same width, so the five chunks are equal-sized -- derive the + // stride from the file rather than hard-coding it. The byte flipped is + // the last one before the chunk's trailing CRC, so that CRC goes stale. Path f = dir.resolve(".symbol-dict"); byte[] bytes = Files.readAllBytes(f); - bytes[23] ^= 0x7F; + int chunkLen = (bytes.length - HEADER_SIZE) / 5; + int chunk2 = HEADER_SIZE + 2 * chunkLen; + bytes[chunk2 + chunkLen - CRC_SIZE - 1] ^= 0x7F; Files.write(f, bytes); PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(re); try { - // Only the intact prefix [s0, s1] is trusted; the corrupt e2 + // Only the intact prefix [s0, s1] is trusted; the corrupt chunk // and everything after it are dropped. No recovered symbol is // the corrupted string -- the tear is DETECTED, never silently // misattributed. - Assert.assertEquals("parse must stop at the corrupt interior entry", 2, re.size()); + Assert.assertEquals("parse must stop at the corrupt interior chunk", 2, re.size()); ObjList s = re.readLoadedSymbols(); Assert.assertEquals("s0", s.getQuick(0)); Assert.assertEquals("s1", s.getQuick(1)); @@ -611,31 +683,40 @@ public void testRemoveOrphanDeletesFile() throws Exception { } @Test - public void testTooLargeToReopenRecreatesEmpty() throws Exception { - // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be read - // into one int-sized buffer; open() must recreate it empty (fail-clean, like - // every other unreadable-file case) rather than truncate the multi-GB file - // via a negative/zero (int) length cast. A fault facade reports the huge - // length without needing a real 2GB file on disk. + public void testTooLargeToReopenDegradesWithoutOpeningTheFile() throws Exception { + // A dictionary at or past Integer.MAX_VALUE cannot be read into one int-sized + // buffer, so open() must short-circuit to null BEFORE touching the file. + // + // The facade reports 2^32 + 100 -- deliberately NOT Integer.MAX_VALUE + 1. + // That value casts to Integer.MIN_VALUE, which Unsafe.malloc rejects anyway, + // so openExisting's catch would produce the same null and the test could not + // tell the guard from its absence (it was exactly that vacuous before). A + // length of 2^32 + k instead casts to a SMALL POSITIVE prefix -- the branch + // the guard's own javadoc calls out -- under which openExisting would happily + // malloc, read a truncated prefix, and parse it. + // + // The assertion that pins the guard is therefore NOT the null (both paths give + // null) but that the file is never even OPENED: with the guard, open() returns + // before openRW; without it, openExisting opens the file and parses garbage. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { - // Seed a small real file so exists() is true and the facade's huge - // length() drives the > Integer.MAX_VALUE reopen guard. PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(seed); - seed.appendSymbol("a"); - seed.close(); - - PersistedSymbolDict d = PersistedSymbolDict.open(new HugeLengthFacade(), dir.toString()); - Assert.assertNotNull(d); - try { - Assert.assertEquals("a >2GB dictionary must recreate empty", 0, d.size()); - d.appendSymbol("X"); - Assert.assertEquals(1, d.size()); - } finally { - d.close(); + for (int i = 0; i < 40; i++) { + seed.appendSymbol("sym" + i); } + seed.close(); + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + + HugeLengthFacade ff = new HugeLengthFacade(); + Assert.assertNull("a >=2GB dictionary must degrade to null", + PersistedSymbolDict.open(ff, dir.toString())); + Assert.assertEquals("the guard must short-circuit BEFORE opening the file", + 0, ff.openRwCalls); + Assert.assertArrayEquals("open() must NOT destroy or trim an oversized dictionary", + before, Files.readAllBytes(f)); } finally { rmDir(dir); } @@ -693,15 +774,19 @@ public void testTornTrailingEntrySelfHeals() throws Exception { } @Test - public void testTruncateFailureRecreatesEmpty() throws Exception { - // A host crash can leave a torn/stale tail past the last complete entry. + public void testTruncateFailureDegradesWithoutDestroyingTheFile() throws Exception { + // A host crash can leave a torn/stale tail past the last complete chunk. // open() drops it with a truncate; if that truncate FAILS (a read-only // remount, a Windows share lock), the file still exposes the stale bytes, - // whose self-consistent per-entry CRC a later shifted parse could accept as - // a real symbol. So a failed truncate must make the file UNTRUSTED -- - // open() recreates it empty (fail-clean) rather than returning a dict laid - // over stale bytes. Drive the truncate failure with a facade and assert the - // reopened dictionary is empty, not the [one, two] prefix. + // whose self-consistent chunk CRC a later shifted parse could accept as a real + // symbol. So a failed truncate must make the file UNTRUSTED -- open() degrades + // to null (the sender falls back to full self-sufficient frames) rather than + // returning a dict laid over stale bytes. + // + // It must NOT recreate the file, which is what it used to do: a read-only + // remount is transient, and destroying the [one, two] prefix on the way past it + // makes every surviving delta frame permanently unreplayable. Degrading leaves + // the bytes for the next attempt, once the mount is writable again. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { @@ -710,26 +795,31 @@ public void testTruncateFailureRecreatesEmpty() throws Exception { d.appendSymbol("two"); d.close(); - // Append a torn trailing record (length prefix 5, only 2 bytes) so - // the reopen parses [one, two], then finds validLen < len and tries - // to truncate the tail -- the branch under test. + // Append a torn trailing record so the reopen parses [one, two], then + // finds validLen < len and tries to truncate the tail -- the branch + // under test. Path f = dir.resolve(".symbol-dict"); long cleanLen = Files.size(f); // header + "one" + "two", no tail Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + byte[] before = Files.readAllBytes(f); // Reopen through a facade whose truncate() fails. - PersistedSymbolDict re = PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString()); + Assert.assertNull("a dict whose torn tail cannot be trimmed must degrade to null", + PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString())); + Assert.assertArrayEquals("a failed truncate must NOT destroy the dictionary", + before, Files.readAllBytes(f)); + + // And once the filesystem is writable again, the SAME file recovers its + // intact prefix in full -- which is the whole point of not destroying it. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(re); try { - // The failed truncate made the file untrusted, so open() recreated - // it empty rather than trusting the [one, two] prefix over a stale - // tail: size()==0, not 2, and no recovered symbols. - Assert.assertEquals("failed truncate must recreate the dictionary empty", 0, re.size()); - Assert.assertEquals(0, re.readLoadedSymbols().size()); - // openFresh rewrote a bare 8-byte header (magic + version), so both - // the two entries and the torn tail are gone. - Assert.assertEquals("recreated file is a bare header", 8L, Files.size(f)); + Assert.assertEquals("the intact prefix survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + Assert.assertEquals("the torn tail is trimmed once truncate works", cleanLen, Files.size(f)); } finally { re.close(); } @@ -894,12 +984,28 @@ public boolean truncate(int fd, long size) { /** * Reports a dictionary length past {@link Integer#MAX_VALUE} -- reproducing a * dictionary that legitimately grew beyond 2GB, which {@code open()} cannot read - * into one int-sized buffer and must recreate empty. + * into one int-sized buffer and must refuse before touching the file. + *

    + * The length is {@code 2^32 + 100}, NOT {@code Integer.MAX_VALUE + 1}. The latter + * casts to {@code Integer.MIN_VALUE}, which {@code Unsafe.malloc} rejects on its + * own, so {@code openExisting}'s catch would produce the same {@code null} with or + * without the guard -- which is precisely what made the old version of this test + * vacuous. {@code 2^32 + k} casts to a small POSITIVE prefix instead, so without + * the guard {@code openExisting} really would open the file and parse a truncated + * prefix of it. {@link #openRwCalls} is what catches that. */ private static final class HugeLengthFacade extends DelegatingFilesFacade { + int openRwCalls; + @Override public long length(String path) { - return (long) Integer.MAX_VALUE + 1L; + return (1L << 32) + 100L; + } + + @Override + public int openRW(String path) { + openRwCalls++; + return super.openRW(path); } } From dd48d4336f9e8f3296dc28ab3e89308b10726c84 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:17:22 +0100 Subject: [PATCH 56/78] Assert the new cap-gap config key is honored WsSenderConfigHonoredTest enforces that every key registered in ConfigSchema is actually plumbed through to the sender. The new catchup_cap_gap_min_escalation_window_millis key needed its assertion. --- .../io/questdb/client/test/impl/WsSenderConfigHonoredTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index dba5765f..f6287825 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -75,6 +75,8 @@ public void testEveryIngressKeyIsHonored() { assertHonored("max_background_drainers=6", "max_background_drainers", 6); assertHonored("max_frame_rejections=6", "max_frame_rejections", 6); assertHonored("poison_min_escalation_window_millis=7500", "poison_min_escalation_window_millis", 7500L); + assertHonored("catchup_cap_gap_min_escalation_window_millis=90000", + "catchup_cap_gap_min_escalation_window_millis", 90000L); assertHonored("error_inbox_capacity=128", "error_inbox_capacity", 128); assertHonored("connection_listener_inbox_capacity=64", "connection_listener_inbox_capacity", 64); assertHonored("token=ey.abc", "token", "ey.abc"); From 521b48d85da397c8bfb33cded799bd8ae4e5c08a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:46:27 +0100 Subject: [PATCH 57/78] Close the untested gaps around the symbol-dict guards Four behaviours the delta symbol-dictionary work relies on had no test that could fail if they regressed. Each mutation below leaves the whole suite green today. Pin the split path's write-ahead ordering. Moving the persist after the publish in flushPendingRowsSplit left every test passing, because the only split-path test checked the dictionary after a SUCCESSFUL flush -- where the symbols land either way. Only a publish that FAILS while the symbols are new can tell the orderings apart, so the new test caps the server batch size to force the split and shrinks the segment so the split frame cannot be published. With the ordering broken, a process crash in that window would leave a recorded frame whose deltaStart exceeds the recovered dictionary, and recovery would quarantine a slot that should have drained cleanly. Pin the persist-failure translation. A short write to .symbol-dict -- a full disk, an exhausted quota -- throws a low-level IllegalStateException that persistNewSymbolsBeforePublish wraps in a LineSenderException. Deleting the wrap left the suite green, yet the raw exception would sail past every caller's catch around flush(). Nothing could reach that path: PersistedSymbolDict has facade-aware overloads but CursorSendEngine called only the FilesFacade.INSTANCE ones, so the engine now takes a FilesFacade and the test drives a real short write through the real producer path. Pin the sealed-segment walk. Deleting the sealed half of SegmentRing.maxSymbolDeltaEnd left the suite green because every test fits its slot in one active segment. It is not a benign wrong answer: recoveredMaxSymbolId collapses to -1, and the seed guard fires on `>= pd.size()`, so a value that is too low never fires at all -- a torn dictionary is trusted and the producer silently reuses ids the surviving frames already define. The walk earns its keep exactly when the active segment holds nothing but an aborted deferred tail, which is the crash the guard was built for, so the test reproduces that shape. Hoist DelegatingFilesFacade into test/tools so a fault seam can be shared rather than stamped into each test that needs one. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 33 +++- .../qwp/client/DeltaDictRecoveryTest.java | 71 ++++++++ .../qwp/client/SelfSufficientFramesTest.java | 85 +++++++++ .../sf/cursor/CursorSendEngineTest.java | 6 +- ...CursorWebSocketSendLoopOrphanTailTest.java | 65 +++++++ .../sf/cursor/PersistedSymbolDictTest.java | 136 +------------- .../test/tools/DelegatingFilesFacade.java | 170 ++++++++++++++++++ 7 files changed, 424 insertions(+), 142 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index a2e20d35..ffd964f3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -27,8 +27,10 @@ import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import java.util.concurrent.locks.LockSupport; @@ -154,9 +156,29 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, FilesFacade.INSTANCE); + } + + /** + * As {@link #CursorSendEngine(String, long, long, long)}, but with an explicit + * {@link FilesFacade} for the persisted symbol dictionary. + *

    + * The seam exists so a test can drive a dictionary I/O fault -- a short write from + * a full disk or an exhausted quota -- through the REAL producer path + * ({@code flush()} -> the write-ahead persist), and assert it surfaces as a + * {@code LineSenderException} like every other flush-path failure rather than as a + * raw {@code IllegalStateException} that would sail past every caller's + * {@code catch (LineSenderException)}. Nothing else could reach that translation: + * {@code PersistedSymbolDict} has facade-aware overloads, but the engine used to + * call only the {@code FilesFacade.INSTANCE} ones, so no fault could be injected + * from outside. + */ + @TestOnly + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, FilesFacade dictFf) { this(sfDir, segmentSizeBytes, new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes), - true, appendDeadlineNanos); + true, appendDeadlineNanos, dictFf); } /** @@ -165,11 +187,12 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, * ownership of the manager. Uses the default append deadline. */ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager) { - this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS); + this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS, + FilesFacade.INSTANCE); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, - boolean ownsManager, long appendDeadlineNanos) { + boolean ownsManager, long appendDeadlineNanos, FilesFacade dictFf) { // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments // live in malloc'd native memory. @@ -278,7 +301,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // Load the persisted symbol dictionary so delta-encoded frames // in this recovered slot can be re-registered on the fresh // server before replay. Null on open failure -> delta disabled. - persistedDictInProgress = PersistedSymbolDict.open(sfDir); + persistedDictInProgress = PersistedSymbolDict.open(dictFf, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -389,7 +412,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // Windows share lock); if the clean open itself fails, // persistedSymbolDict stays null and the sender falls back to full // self-sufficient frames, which is also safe. - persistedDictInProgress = PersistedSymbolDict.openClean(sfDir); + persistedDictInProgress = PersistedSymbolDict.openClean(dictFf, sfDir); } MmapSegment initial; String initialPath = null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 31153408..4be99883 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -26,6 +26,9 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -580,6 +583,56 @@ public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Excepti }); } + @Test + public void testPersistFailureSurfacesAsLineSenderException() throws Exception { + // A dictionary write that fails mid-flush -- a full disk, an exhausted quota -- + // must reach the caller as a LineSenderException, like every other flush-path + // failure. PersistedSymbolDict throws a low-level IllegalStateException on a short + // write; persistNewSymbolsBeforePublish wraps it. Without the wrap the raw + // IllegalStateException sails straight past every user's + // `catch (LineSenderException)` around flush() and takes the application down. + // + // Nothing could reach that translation before: PersistedSymbolDict has + // facade-aware overloads, but CursorSendEngine called only the + // FilesFacade.INSTANCE ones, so no test could inject a dictionary I/O fault + // through the real producer path. The engine now takes a FilesFacade for exactly + // this, and the sender is built on that engine directly -- the same + // QwpWebSocketSender.connect(..., CursorSendEngine) entry point Sender.build uses. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String slot = Paths.get(sfDir, "default").toString(); + Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir, + io.questdb.client.std.Files.DIR_MODE_DEFAULT)); + ShortDictWriteFacade ff = new ShortDictWriteFacade(); + // The engine owns the dictionary; the fault facade reaches only it, so the + // segment files still write normally and the ONLY failure is the persist. + CursorSendEngine engine = new CursorSendEngine( + slot, 4L * 1024 * 1024, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff); + try (Sender sender = QwpWebSocketSender.connect( + "localhost", port, null, 0, 0, 0L, null, false, engine)) { + ff.armed = true; // the next dictionary append short-writes + sender.table("m").symbol("s", "boom").longColumn("v", 1L).atNow(); + try { + sender.flush(); + Assert.fail("a short write to the symbol dictionary must fail the flush"); + } catch (LineSenderException expected) { + Assert.assertTrue("the persist failure must be reported as a sender error, " + + "not leak a raw IllegalStateException: " + expected.getMessage(), + expected.getMessage().contains("failed to persist symbol dictionary before publish")); + } + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered row; the facade has disarmed, so + // this normally succeeds. Either way it is not what we assert. + } + } + }); + } + @Test public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs @@ -1256,4 +1309,22 @@ private static byte[] buildAck(long seq) { return buf; } } + /** + * Short-writes the FIRST dictionary append after it is armed, modelling a disk that + * fills mid-flush. Offset 0 is left alone so the file header still writes -- only the + * entry append fails, which is the path under test. + */ + private static final class ShortDictWriteFacade extends DelegatingFilesFacade { + boolean armed; + + @Override + public long write(int fd, long addr, long len, long offset) { + if (armed && offset > 0 && len > 1) { + armed = false; + return INSTANCE.write(fd, addr, len - 1, offset); + } + return INSTANCE.write(fd, addr, len, offset); + } + } + } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 5729aa44..c0d27e09 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -26,6 +26,8 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.ObjList; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -332,6 +334,89 @@ public void testOversizedTableSplitStrandsNothing() throws Exception { }); } + @Test + public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception { + // The SPLIT path's write-ahead ordering: persistNewSymbolsBeforePublish must run + // BEFORE sealAndSwapBuffer publishes the frame to the ring, exactly as it does on + // the non-split path. + // + // Its sibling below (testFileModeSplitPersistsDictBeforePublish) checks the dict + // after a SUCCESSFUL flush, which proves nothing about ORDER -- the symbols land + // in the file either way. Only a publish that FAILS while the symbols are new can + // tell the two apart: with the write-ahead intact the symbols are already durable + // when the publish throws; with the persist moved after the publish, the throw + // beats it and the dictionary is still empty. (The non-split path is pinned this + // way by DeltaDictRecoveryTest.testFailedPublishDoesNotDuplicatePersistedSymbols; + // the split path had no equivalent.) + // + // Why it matters: store-and-forward is process-crash durable. If a split frame + // reaches the ring before its symbols reach .symbol-dict, a JVM crash in that + // window leaves a recorded frame whose deltaStart exceeds the recovered + // dictionary -- so recovery declares the slot unreplayable and quarantines a slot + // that should have drained cleanly. + // + // Setup: the server cap (150) splits the two-table batch, and each split frame + // still FITS that cap, so the split pre-flight passes and the publish is actually + // attempted. sf_max_bytes=64 then makes every frame larger than the segment, so + // appendBlocking fails with PAYLOAD_TOO_LARGE -- deterministically, no + // backpressure timing needed. + Path sfDir = Files.createTempDirectory("qwp-sf-split-persist-fail"); + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=64;"; + String pad = TestUtils.repeat("x", 60); + Sender sender = Sender.fromConfig(config); + try { + sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); + try { + sender.flush(); + Assert.fail("a split frame larger than the segment must fail to publish"); + } catch (LineSenderException expected) { + // PAYLOAD_TOO_LARGE -- the publish, not the pre-flight. + } + Assert.assertEquals("the publish must fail, so no frame reaches the server", + 0, handler.batches.size()); + + // The write-ahead already ran: both of the batch's new symbols are + // durable even though the frame that references them never + // published. Move the persist after sealAndSwapBuffer and this is 0. + PersistedSymbolDict pd = PersistedSymbolDict.open( + sfDir.resolve("default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("the split path must persist its new symbols BEFORE " + + "publishing the frame that references them", 2, pd.size()); + ObjList symbols = pd.readLoadedSymbols(); + Assert.assertEquals("alpha", symbols.getQuick(0)); + Assert.assertEquals("bravo", symbols.getQuick(1)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized rows and fails + // again; expected, and not what this test asserts. close() still + // runs its resource cleanup, so no native memory leaks. + } + } + } + }); + } finally { + rmDir(sfDir); + } + } + @Test public void testFileModeSplitPersistsDictBeforePublish() throws Exception { // File-mode store-and-forward + a SPLIT flush: a two-table batch whose combined diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index f4de1ff2..0e874ba5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -31,6 +31,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; @@ -608,11 +609,12 @@ private static void assertSlotCanBeReacquired(String sfDir) { private static Throwable invokeOwnedPrivateConstructorExpectingFailure( String sfDir, long segmentSizeBytes, SegmentManager manager) throws Exception { Constructor ctor = CursorSendEngine.class.getDeclaredConstructor( - String.class, long.class, SegmentManager.class, boolean.class, long.class); + String.class, long.class, SegmentManager.class, boolean.class, long.class, + FilesFacade.class); ctor.setAccessible(true); try { ctor.newInstance(sfDir, segmentSizeBytes, manager, true, - CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); + CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, FilesFacade.INSTANCE); fail("expected constructor failure"); return null; } catch (InvocationTargetException e) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index bf16f3b0..d19f0e12 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -44,6 +44,8 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -223,6 +225,69 @@ public void testSlowPathReplaysBelowTailThenRetiresAndRecyclesOnce() throws Exce }); } + @Test + public void testRecoveredMaxSymbolIdSpansSealedSegments() throws Exception { + // The torn-dict detector must walk the SEALED segments too, not just the active + // one. Every other test in the suite fits its whole slot in a single active + // segment, so the sealed walk in SegmentRing.maxSymbolDeltaEnd was dead code as + // far as the tests were concerned: deleting it left them all green while + // recoveredMaxSymbolId silently collapsed to -1. + // + // -1 is not a benign wrong answer. seedGlobalDictionaryFromPersisted's guard is + // `recoveredMaxSymbolId >= pd.size()`, so a value that is too LOW never fires: a + // torn dictionary is trusted, the producer resumes seeded from the short + // dictionary, and it hands ids the surviving frames already define to different + // symbols -- the exact silent misattribution the whole feature exists to prevent. + // + // The shape that needs the sealed walk is the crash the guard was built for. In a + // plain multi-segment slot the ACTIVE segment holds the newest frames and thus the + // highest ids, so an active-only walk happens to get the right answer. It stops + // being right when the active segment contributes NOTHING to the committed range + // -- here, a producer that died mid-transaction, leaving an aborted deferred tail + // long enough to overflow into a fresh segment. maxSymbolDeltaEnd skips those + // frames (fsn > recoveredCommitBoundaryFsn), so the highest COMMITTED id is left + // sitting in a SEALED segment, reachable only through the sealed walk. + TestUtils.assertMemoryLeak(() -> { + // Small segments so the tail really does roll one. Each frame is + // FRAME_HEADER_SIZE + (QWP HEADER_SIZE + 2) = 22 bytes, against 256 - 24 + // usable, so a segment holds ~10. + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 256)) { + // Committed delta frames registering ids 0,1,2 -- the highest COMMITTED + // ids in the slot. + for (int i = 0; i < 3; i++) { + appendDeltaFrame(engine, false, i, 1); + } + assertNull("the committed frames must still be in the ACTIVE segment here", + engine.firstSealed()); + // The aborted transaction: deferred frames with no covering commit. Keep + // appending until they overflow the segment, then a few more, so the + // committed frames end up SEALED and the active segment holds nothing but + // the tail. Ids stay small so the test's one-byte varint encoding holds. + int deferredId = 3; + for (int i = 0; i < 64 && engine.firstSealed() == null; i++) { + appendDeltaFrame(engine, true, deferredId++, 1); + } + assertNotNull("the deferred tail must have rolled a segment", engine.firstSealed()); + for (int i = 0; i < 3; i++) { + appendDeltaFrame(engine, true, deferredId++, 1); + } + } + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 256)) { + assertTrue(engine.wasRecoveredFromDisk()); + assertNotNull("the recovered slot must have a sealed segment", engine.firstSealed()); + assertEquals("the last commit-bearing frame is fsn 2", + 2L, engine.recoveredCommitBoundaryFsn()); + // ids 0,1,2 were introduced by the committed frames, all of which now live + // in a SEALED segment. The active segment holds only the deferred tail, + // which the walk skips -- so an active-only walk returns 0 and this comes + // back -1. + assertEquals("the highest COMMITTED id lives in a SEALED segment; " + + "maxSymbolDeltaEnd must walk the sealed segments to see it", + 2L, engine.recoveredMaxSymbolId()); + } + }); + } + @Test public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception { // recoveredMaxSymbolId must reflect only COMMITTED (transmitted) frames, not diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index b1255aac..c5222e0e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; +import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -833,141 +834,6 @@ private static void rmDir(Path dir) { TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } - /** - * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE}. - * Subclasses inject a single fault; every other call hits the real filesystem. - */ - private static class DelegatingFilesFacade implements FilesFacade { - @Override - public long allocNativePath(String path) { - return INSTANCE.allocNativePath(path); - } - - @Override - public boolean allocate(int fd, long size) { - return INSTANCE.allocate(fd, size); - } - - @Override - public int close(int fd) { - return INSTANCE.close(fd); - } - - @Override - public boolean exists(String path) { - return INSTANCE.exists(path); - } - - @Override - public void findClose(long findPtr) { - INSTANCE.findClose(findPtr); - } - - @Override - public long findFirst(String dir) { - return INSTANCE.findFirst(dir); - } - - @Override - public long findName(long findPtr) { - return INSTANCE.findName(findPtr); - } - - @Override - public int findNext(long findPtr) { - return INSTANCE.findNext(findPtr); - } - - @Override - public int findType(long findPtr) { - return INSTANCE.findType(findPtr); - } - - @Override - public void freeNativePath(long pathPtr) { - INSTANCE.freeNativePath(pathPtr); - } - - @Override - public int fsync(int fd) { - return INSTANCE.fsync(fd); - } - - @Override - public long length(int fd) { - return INSTANCE.length(fd); - } - - @Override - public long length(String path) { - return INSTANCE.length(path); - } - - @Override - public long length(long pathPtr) { - return INSTANCE.length(pathPtr); - } - - @Override - public int lock(int fd) { - return INSTANCE.lock(fd); - } - - @Override - public int mkdir(String path, int mode) { - return INSTANCE.mkdir(path, mode); - } - - @Override - public int openCleanRW(String path) { - return INSTANCE.openCleanRW(path); - } - - @Override - public int openCleanRW(long pathPtr) { - return INSTANCE.openCleanRW(pathPtr); - } - - @Override - public int openRW(String path) { - return INSTANCE.openRW(path); - } - - @Override - public int openRW(long pathPtr) { - return INSTANCE.openRW(pathPtr); - } - - @Override - public long read(int fd, long addr, long len, long offset) { - return INSTANCE.read(fd, addr, len, offset); - } - - @Override - public boolean remove(String path) { - return INSTANCE.remove(path); - } - - @Override - public boolean remove(long pathPtr) { - return INSTANCE.remove(pathPtr); - } - - @Override - public int rename(String oldPath, String newPath) { - return INSTANCE.rename(oldPath, newPath); - } - - @Override - public boolean truncate(int fd, long size) { - return INSTANCE.truncate(fd, size); - } - - @Override - public long write(int fd, long addr, long len, long offset) { - return INSTANCE.write(fd, addr, len, offset); - } - } /** * Fails every {@link #truncate(int, long)} -- reproducing a host where the diff --git a/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java new file mode 100644 index 00000000..516800c4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java @@ -0,0 +1,170 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.tools; + +import io.questdb.client.std.FilesFacade; + +/** + * Pass-through {@link FilesFacade} that forwards every call to + * {@link FilesFacade#INSTANCE}. Subclass it and override the ONE method whose + * failure you want to inject -- a short write, a truncate that cannot trim, an + * implausible file length -- so a test can drive a real I/O failure path without a + * real broken disk. + *

    + * {@code FilesFacade} has no default methods, so the delegation boilerplate is + * unavoidable; it lives here once rather than being stamped into each test that + * needs a fault seam. + */ +public class DelegatingFilesFacade implements FilesFacade { + @Override + public long allocNativePath(String path) { + return INSTANCE.allocNativePath(path); + } + + @Override + public boolean allocate(int fd, long size) { + return INSTANCE.allocate(fd, size); + } + + @Override + public int close(int fd) { + return INSTANCE.close(fd); + } + + @Override + public boolean exists(String path) { + return INSTANCE.exists(path); + } + + @Override + public void findClose(long findPtr) { + INSTANCE.findClose(findPtr); + } + + @Override + public long findFirst(String dir) { + return INSTANCE.findFirst(dir); + } + + @Override + public long findName(long findPtr) { + return INSTANCE.findName(findPtr); + } + + @Override + public int findNext(long findPtr) { + return INSTANCE.findNext(findPtr); + } + + @Override + public int findType(long findPtr) { + return INSTANCE.findType(findPtr); + } + + @Override + public void freeNativePath(long pathPtr) { + INSTANCE.freeNativePath(pathPtr); + } + + @Override + public int fsync(int fd) { + return INSTANCE.fsync(fd); + } + + @Override + public long length(int fd) { + return INSTANCE.length(fd); + } + + @Override + public long length(String path) { + return INSTANCE.length(path); + } + + @Override + public long length(long pathPtr) { + return INSTANCE.length(pathPtr); + } + + @Override + public int lock(int fd) { + return INSTANCE.lock(fd); + } + + @Override + public int mkdir(String path, int mode) { + return INSTANCE.mkdir(path, mode); + } + + @Override + public int openCleanRW(String path) { + return INSTANCE.openCleanRW(path); + } + + @Override + public int openCleanRW(long pathPtr) { + return INSTANCE.openCleanRW(pathPtr); + } + + @Override + public int openRW(String path) { + return INSTANCE.openRW(path); + } + + @Override + public int openRW(long pathPtr) { + return INSTANCE.openRW(pathPtr); + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + + @Override + public boolean remove(String path) { + return INSTANCE.remove(path); + } + + @Override + public boolean remove(long pathPtr) { + return INSTANCE.remove(pathPtr); + } + + @Override + public int rename(String oldPath, String newPath) { + return INSTANCE.rename(oldPath, newPath); + } + + @Override + public boolean truncate(int fd, long size) { + return INSTANCE.truncate(fd, size); + } + + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } +} From aeb92b559467fbeb1c4516e5ac6dc6c8fffa80c3 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 20:14:10 +0100 Subject: [PATCH 58/78] Set aside the slots recovery cannot rescue, instead of failing The full-dictionary rebuild and the unreplayable-slot quarantine answer two halves of the same question and have to be ordered, not merged. Rebuilding the dictionary from the surviving frames' own delta sections rescues most of the slots that used to fail, so it must run FIRST -- a quarantine that pre-judged the slot on its own coverage arithmetic would set aside slots recovery can still replay in full, and strand data that drains perfectly. What it cannot rescue it still throws on, and that throw still escaped build(). senderId is stable and a not-fully-drained slot is retained on close, so every retry re-recovered the same slot and threw again: the application could not construct a Sender at all, and so could not even BUFFER new rows. One already-lost batch became an unbounded outage of everything after it, which inverts the guarantee store-and-forward exists to give. So seedGlobalDictionaryFromPersisted is now the single authority on whether a slot is replayable -- it is the only code that has tried every source of truth -- and its give-up is typed. build() waits for that verdict rather than forming its own, and on it sets the slot aside: the bytes are kept for forensics and resend, the .failed sentinel tells the orphan drainer the copy is human-in-the-loop, and the producer continues on a clean slot. The engine's own isRecoveredDictionaryIncomplete() guess goes away with it. Not one frame of a set-aside slot ever reaches the wire; that safety property is unchanged and still asserted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../main/java/io/questdb/client/Sender.java | 62 ++++++--- .../qwp/client/QwpWebSocketSender.java | 6 +- .../client/sf/cursor/CursorSendEngine.java | 31 ----- .../sf/cursor/UnreplayableSlotException.java | 51 ++++++++ .../qwp/client/DeltaDictRecoveryTest.java | 123 ++++++++++++++++-- 5 files changed, 214 insertions(+), 59 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 447d5fad..b674cf1b 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -40,6 +40,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -1560,11 +1561,6 @@ public Sender build() { CursorSendEngine cursorEngine = new CursorSendEngine( slotPath, actualSfMaxBytes, actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - if (cursorEngine.isRecoveredDictionaryIncomplete()) { - cursorEngine = quarantineTornSlot( - cursorEngine, sfDir, senderId, slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - } int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; @@ -1576,7 +1572,15 @@ public Sender build() { for (int i = 0, n = hosts.size(); i < n; i++) { wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); } - QwpWebSocketSender connected; + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + QwpWebSocketSender connected = null; + boolean quarantined = false; + while (connected == null) { try { connected = QwpWebSocketSender.connect( wsEndpoints, @@ -1603,6 +1607,32 @@ public Sender build() { actualPoisonMinEscalationWindowMillis, actualCatchUpCapGapMinEscalationWindowMillis ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); } catch (Throwable t) { // connect() failed before ownership of cursorEngine // transferred — close it ourselves. @@ -1613,6 +1643,7 @@ public Sender build() { } throw t; } + } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY // failure must close `connected` (which closes the engine @@ -3001,18 +3032,17 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na * throw -- loudly, and never silently dropping bytes. */ private static CursorSendEngine quarantineTornSlot( - CursorSendEngine torn, String sfDir, String senderId, String slotPath, + CursorSendEngine torn, UnreplayableSlotException cause, String sfDir, + String senderId, String slotPath, long sfMaxBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos ) { - long recoveredMaxSymbolId = torn.recoveredMaxSymbolId(); - PersistedSymbolDict dict = torn.getPersistedSymbolDict(); - int coverage = dict != null ? dict.size() : 0; - String detail = "recovered store-and-forward symbol dictionary cannot cover the surviving " - + "frames (likely a host crash tore its unsynced tail): frames reference symbol id " - + recoveredMaxSymbolId + " but the recovered dictionary holds only " + coverage - + " id(s)"; - // Release the slot lock and the dictionary fd before renaming. The slot is not - // fully drained, so close() retains every file. + // The verdict, and the reason, come from the recovery seed -- the only code that + // has tried every source of truth. Recomputing them here would mean a second, + // independently-drifting notion of "unreplayable". + final String detail = cause.getMessage(); + // Release the slot lock and the dictionary fd before renaming. connect()'s failure + // path already closed the engine; close() is idempotent, so make it explicit rather + // than depend on that. torn.close(); String quarantinePath = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index fb9a7f52..47319455 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -47,6 +47,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; @@ -3926,7 +3927,10 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { ObjList fromFrames = new ObjList<>(); long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames); if (coverage < 0) { - throw new LineSenderException( + // Typed, because Sender.build() sets such a slot aside instead of failing: this + // is the point at which every source of truth has been tried and none of them + // holds the missing ids. See UnreplayableSlotException. + throw new UnreplayableSlotException( "recovered store-and-forward symbol dictionary is incomplete and cannot be " + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + "tail): the frames reference symbol ids below their own delta start, which " diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index ffd964f3..9f3add89 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -767,37 +767,6 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } - /** - * Whether this RECOVERED slot's surviving frames reference symbol ids the - * recovered dictionary cannot cover -- i.e. the slot cannot be drained by a - * producer that shares its id space, and a producer seeded from the short - * dictionary would reuse ids those frames already define. - *

    - * True in two situations, both of which mean the same thing to the caller: - *

      - *
    • the dictionary opened but is SHORT of the frames (a host/power crash - * tore its unsynced tail -- see {@code PersistedSymbolDict}'s durability - * note); or
      • - *
    • the dictionary did not open at all ({@code persistedSymbolDict == null} - * -- an unreadable or torn side-file, fd exhaustion, a read-only remount) - * while the surviving frames are DELTA frames that need one. Coverage is - * then zero, so any frame referencing an id at all is unreplayable.
      • - *
    - * Always false for a fresh (non-recovered) slot: {@code recoveredMaxSymbolId} is - * -1 there, and -1 is below every coverage. - *

    - * The caller must NOT drain such a slot with a live producer attached. The - * foreground sender quarantines it and starts on a fresh slot - * ({@code Sender.build()}); the orphan drainer reaches the same verdict - * independently via the send loop's replay guard and marks the slot failed. - * Either way the recorded bytes stay on disk and the affected data must be - * resent -- but the producer keeps producing, which is the whole point of - * store-and-forward. - */ - public boolean isRecoveredDictionaryIncomplete() { - long coverage = persistedSymbolDict != null ? persistedSymbolDict.size() : 0; - return recoveredMaxSymbolId >= coverage; - } /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java new file mode 100644 index 00000000..00c0d351 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** + * A recovered store-and-forward slot whose symbol dictionary cannot be rebuilt at all -- + * neither from the persisted side-file's intact prefix nor from the surviving frames' own + * delta sections. Its frames reference ids that nothing still holds, so replaying them + * would null-pad the gap on the server and silently misattribute symbol values. + *

    + * This is the ONE verdict {@code seedGlobalDictionaryFromPersisted} reaches after it has + * tried every source of truth it has, so it is the only signal that may quarantine a slot. + * A distinct type, rather than a message match, because the difference between "this slot + * is unreplayable" and any other {@link LineSenderException} out of the connect path is the + * difference between setting a slot aside and silently discarding one that was fine. + *

    + * It stays a {@code LineSenderException} so that a caller which does NOT handle it -- the + * background drainer, a test constructing the sender directly -- keeps the old fail-clean + * behaviour rather than seeing a new checked type. Only {@code Sender.build()} treats it as + * recoverable, by setting the slot aside and starting the producer on a fresh one. + */ +public class UnreplayableSlotException extends LineSenderException { + + public UnreplayableSlotException(CharSequence message) { + super(message); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 4be99883..3b94d171 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -155,6 +155,71 @@ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Excep }); } + @Test + public void testUnreplayableSlotIsSetAsideAndTheProducerKeepsProducing() throws Exception { + // The point of setting an unreplayable slot aside rather than failing: the + // producer stays ALIVE and its new data still reaches the server. This is the + // property store-and-forward exists to guarantee, and the one bricking build() + // destroyed -- one host crash and the application could never construct a Sender + // again, so every row after the tear was lost too, not just the torn batch. + // + // Asserted end to end against a real wire: the fresh rows arrive and the server's + // reconstructed dictionary holds EXACTLY the post-recovery symbols. Not one + // symbol from the unreplayable slot appears -- so the producer kept producing + // AND the torn frames stayed off the wire. + assertMemoryLeak(() -> { + // Phase 1: record delta frames, then lose the whole dictionary (host crash). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "torn-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.write(dict, + Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); // header only: 0 symbols + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: the recovering sender sets the slot aside and keeps producing. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "fresh-a").longColumn("v", 100).atNow(); + s2.table("m").symbol("s", "fresh-b").longColumn("v", 101).atNow(); + s2.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.dictSnapshot().size() < 2) { + Thread.sleep(20); + } + Assert.assertEquals("the new rows must reach the server", + 2, handler.dictSnapshot().size()); + } + + assertUnreplayableSlotSetAside(); + List dict2 = handler.dictSnapshot(); + Assert.assertEquals("the server must hold exactly the post-recovery symbols", + Arrays.asList("fresh-a", "fresh-b"), dict2); + for (String s : dict2) { + Assert.assertFalse("no symbol of the unreplayable slot may reach the server: " + s, + s.startsWith("torn-")); + } + } + }); + } + @Test public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { assertMemoryLeak(() -> { @@ -196,19 +261,20 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - try { - Sender s2 = Sender.fromConfig(cfg); - s2.close(); - } catch (LineSenderException e) { - terminal = e; - } + // The dictionary cannot be rebuilt from any source, so this slot is genuinely + // unreplayable -- and build() SETS IT ASIDE rather than failing. Failing here + // is what bricked the sender: senderId is stable and a not-fully-drained slot + // is retained on close, so every retry re-recovered the same slot and threw + // again, and the application could not construct a Sender at all -- it could + // not even buffer new rows. Now the producer comes up on a clean slot and the + // unreplayable bytes are kept aside for resend. + Sender.fromConfig(cfg).close(); + + // The safety property is unchanged: not one frame of the unreplayable slot + // may reach the server. Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", 0, handler.frames.get()); - Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("cannot be rebuilt from the surviving frames") - && terminal.getMessage().contains("resend")); + assertUnreplayableSlotSetAside(); } }); } @@ -1327,4 +1393,39 @@ public long write(int fd, long addr, long len, long offset) { } } + + /** + * The unreplayable-slot contract: a recovered slot whose symbol dictionary cannot be + * rebuilt -- not from its own intact prefix, not from the surviving frames' delta + * sections -- is SET ASIDE, never silently drained and never allowed to brick the + * sender. + *

    + * The bytes are kept for forensics and resend, the {@code .failed} sentinel tells the + * orphan drainer to treat the copy as human-in-the-loop rather than retry it forever, + * and the producer continues on a fresh, empty slot. + */ + private void assertUnreplayableSlotSetAside() { + java.nio.file.Path aside = Paths.get(sfDir, "default.unreplayable-0"); + Assert.assertTrue("the unreplayable slot must be set aside, not deleted: " + aside, + java.nio.file.Files.isDirectory(aside)); + Assert.assertTrue("the set-aside slot must keep its recorded frames for resend", + hasSegmentFile(aside)); + Assert.assertTrue("the set-aside slot must carry the .failed sentinel", + java.nio.file.Files.exists(aside.resolve(".failed"))); + Assert.assertTrue("the sender must continue on a live slot", + java.nio.file.Files.isDirectory(Paths.get(sfDir, "default"))); + } + + private static boolean hasSegmentFile(java.nio.file.Path dir) { + java.io.File[] files = dir.toFile().listFiles(); + if (files != null) { + for (java.io.File f : files) { + if (f.getName().endsWith(".sfa") && f.length() > 0) { + return true; + } + } + } + return false; + } + } From ff53e1f0a284b82c4b0f1f86c08e18856a2a0fdc Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 21:00:11 +0100 Subject: [PATCH 59/78] Rebuild the dictionary from the frames still on disk A slot whose symbol dictionary was damaged was being condemned as unreplayable while every symbol it needed was sitting on disk. The sender told the user to resend data that had never been lost, and after the previous commit it set the slot aside instead. Three things kept the rebuild from ever running. setCursorEngine gated the producer's seed on deltaDictEnabled. That flag is false exactly when the slot's dictionary failed to open -- fd exhaustion, a read-only remount, ENOSPC -- which is precisely when the frames on disk are the only surviving copy of the symbols. The rebuild was dead code for the very case it was written for. collectReplaySymbolsAbove walked from ackedFsn+1, so it skipped the acked frames. But an acked frame is not trimmed the instant it is acked: trim unlinks whole SEALED segments. The frames that registered the early ids are almost always still there, and they are exactly the ids the replay set's deltas start above. It walks every frame on disk now, and a slot whose registering frames really HAVE been trimmed still reports the gap, because then nothing holds them. And the send loop's mirror seeded only from the dictionary file, never from the frames -- so even with the other two fixed, the loop would still condemn (deltaStart > sentDictCount) a slot the producer had just seeded successfully. The mirror is the loop's model of what the SERVER holds and it is what the catch-up ships, so it now seeds from the same two sources in the same order, and the two land on the same number by construction. Only when a surviving frame actually depends on ids it does not carry: at maxSymbolDeltaStart == 0 every frame re-registers from id 0 as it replays, so a full-dict slot stays catch-up-free. The recovery tests now assert the strong form -- the fresh server's reconstructed dictionary must come back COMPLETE and in order, which can only happen if the catch-up carried the ids the acked frames still hold. A null-padded or shifted dictionary fails there, which is the corruption the old guard condemned these slots to avoid. It never had to. The quarantine now covers only what is genuinely unrecoverable: the registering frames trimmed away and the dictionary torn, so nothing anywhere holds those ids. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 6 +- .../client/sf/cursor/CursorSendEngine.java | 40 +- .../sf/cursor/CursorWebSocketSendLoop.java | 38 ++ .../qwp/client/DeltaDictRecoveryTest.java | 376 +++++++----------- 4 files changed, 216 insertions(+), 244 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 47319455..d7ba19cf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2258,7 +2258,11 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { // persisted dictionary so newly ingested symbols continue from the // recovered ids (rather than colliding with them at 0), and the delta // baseline resumes where the crashed session left off. - if (deltaDictEnabled && engine.wasRecoveredFromDisk()) { + // NOT gated on deltaDictEnabled. That flag is false exactly when the slot's dictionary + // failed to open -- which is precisely when the frames on disk are the only surviving + // copy of the symbols and the rebuild matters most. Gating the seed on it made the + // rebuild dead code for the very case it was written for. + if (engine != null && engine.wasRecoveredFromDisk()) { seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 9f3add89..37adf4a0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -108,6 +108,14 @@ public final class CursorSendEngine implements QuietCloseable { // against the recovered dictionary size to fail clean instead. Computed once // in the constructor's recovery branch; -1 elsewhere. private long recoveredMaxSymbolId = -1L; + // Highest deltaStart across the recovered COMMITTED frames; 0 when none carries a symbol + // dictionary. ZERO means every surviving frame is SELF-SUFFICIENT -- it re-registers its + // dictionary from id 0 -- so the slot replays with no dictionary at all and the send loop + // needs no catch-up. ABOVE zero means at least one frame is a true delta whose ids depend + // on registrations it does not itself carry, so the loop must seed its mirror (and ship a + // catch-up) before replaying. Both the full-dict-fallback discard below and the send + // loop's mirror seeding key off this. + private long recoveredMaxSymbolDeltaStart; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -380,14 +388,15 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // dictionary. The recoveredMaxSymbolId >= size guard means this never // fires for a slot whose dictionary is intact, nor for an empty slot // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. - if (persistedDictInProgress != null - && recoveredMaxSymbolId >= persistedDictInProgress.size() - && recovered.maxSymbolDeltaStart( + this.recoveredMaxSymbolDeltaStart = recovered.maxSymbolDeltaStart( QwpConstants.MAGIC_MESSAGE, QwpConstants.HEADER_OFFSET_FLAGS, QwpConstants.FLAG_DELTA_SYMBOL_DICT, QwpConstants.HEADER_SIZE, - recoveredCommitBoundaryFsn) == 0L) { + recoveredCommitBoundaryFsn); + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recoveredMaxSymbolDeltaStart == 0L) { persistedDictInProgress.close(); persistedDictInProgress = null; } @@ -715,7 +724,14 @@ public long collectReplaySymbolsAbove(int baseline, ObjList out) { QwpConstants.HEADER_OFFSET_FLAGS, QwpConstants.FLAG_DELTA_SYMBOL_DICT, QwpConstants.HEADER_SIZE, - ackedFsn() + 1L, + // Walk from the LOWEST frame still on disk, not from ackedFsn+1. An acked frame + // is not trimmed the instant it is acked -- trim drops whole SEALED segments -- + // so the symbols it registered are usually still sitting right there, and they + // are exactly the ids the replay set's deltas start above. Skipping them threw + // away the only surviving copy of those symbols and condemned a slot that had + // everything it needed. A slot whose registering frames really HAVE been + // trimmed still reports the gap, because then no frame on disk holds them. + 0L, recoveredCommitBoundaryFsn, baseline, out @@ -842,6 +858,20 @@ public long recoveredMaxSymbolId() { return recoveredMaxSymbolId; } + /** + * Highest {@code deltaStart} across the recovered committed frames; {@code 0} when every + * surviving frame is self-sufficient (or none carries a dictionary at all). + *

    + * The send loop uses this to decide whether it needs a catch-up: at zero, every frame + * re-registers its dictionary from id 0 as it replays, so seeding the mirror -- and + * shipping a catch-up frame off it -- would be pure redundancy. Above zero, at least one + * frame's delta starts above ids it does not itself carry, so the mirror must hold those + * ids before the replay begins or the server null-pads the hole. + */ + public long recoveredMaxSymbolDeltaStart() { + return recoveredMaxSymbolDeltaStart; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 8e34011d..cf3f1637 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -43,6 +43,8 @@ import io.questdb.client.std.CharSequenceLongHashMap; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.str.Utf8s; +import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; @@ -699,6 +701,25 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, sentDictCount = pd.size(); } } + // ...and then from the surviving frames' own delta sections, exactly as the producer + // seeds its dictionary. The mirror is the loop's model of what the SERVER holds and it + // is what the catch-up frame ships, so when it stops at the persisted prefix while the + // producer's baseline runs past it, the loop condemns (deltaStart > sentDictCount) a + // slot the producer just seeded successfully. Same two sources, same order, so the two + // land on the same number by construction. + // ...but ONLY when a surviving frame actually depends on ids it does not carry + // (recoveredMaxSymbolDeltaStart > 0). At zero every frame is self-sufficient and + // re-registers its dictionary from id 0 as it replays, so seeding the mirror would buy + // nothing and cost a catch-up frame on every connection -- full-dict slots must stay + // catch-up-free. + if (engine.recoveredMaxSymbolDeltaStart() > 0L) { + ObjList fromFrames = new ObjList<>(); + if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { + for (int i = 0, n = fromFrames.size(); i < n; i++) { + appendSymbolToMirror(fromFrames.getQuick(i)); + } + } + } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; @@ -2268,6 +2289,23 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart sentDictCount += (int) newCount; } + /** + * Appends one symbol to the sent-dictionary mirror in wire form + * ({@code [len varint][utf8]}), advancing {@code sentDictCount}. Seeds the mirror at + * construction; {@link #accumulateSentDict} extends it from live frames thereafter. + */ + private void appendSymbolToMirror(CharSequence symbol) { + int utf8Len = Utf8s.utf8Bytes(symbol); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; + ensureSentDictCapacity((long) sentDictBytesLen + wireLen); + long p = NativeBufferWriter.writeVarint(sentDictBytesAddr + sentDictBytesLen, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + sentDictBytesLen += wireLen; + sentDictCount++; + } + private void ensureSentDictCapacity(long required) { if (sentDictBytesCapacity >= required) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 3b94d171..aaaeb66a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -156,210 +156,137 @@ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Excep } @Test - public void testUnreplayableSlotIsSetAsideAndTheProducerKeepsProducing() throws Exception { - // The point of setting an unreplayable slot aside rather than failing: the - // producer stays ALIVE and its new data still reaches the server. This is the - // property store-and-forward exists to guarantee, and the one bricking build() - // destroyed -- one host crash and the application could never construct a Sender - // again, so every row after the tear was lost too, not just the torn batch. + public void testTornDictionaryRebuildsFromFramesAcrossTheAckWatermark() throws Exception { + // A host crash truncates the dictionary to nothing, and the ack watermark sits + // mid-stream so the replay set starts at a frame whose delta begins at id 3. // - // Asserted end to end against a real wire: the fresh rows arrive and the server's - // reconstructed dictionary holds EXACTLY the post-recovery symbols. Not one - // symbol from the unreplayable slot appears -- so the producer kept producing - // AND the torn frames stayed off the wire. + // The old guard called that unreplayable and told the user to resend. It is not: ids + // 0..2 are still sitting in the ACKED frames on disk. Trim unlinks whole SEALED + // segments, not individual acked frames, so those six tiny frames are all still in + // the one active segment. Rebuild from them and the slot drains in full. assertMemoryLeak(() -> { - // Phase 1: record delta frames, then lose the whole dictionary (host crash). - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "torn-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } + recordSixDeltaFrames(); java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); java.nio.file.Files.write(dict, - Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); // header only: 0 symbols + Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); // header only writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: the recovering sender sets the slot aside and keeps producing. - DictReconstructingHandler handler = new DictReconstructingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender s2 = Sender.fromConfig(cfg)) { - s2.table("m").symbol("s", "fresh-a").longColumn("v", 100).atNow(); - s2.table("m").symbol("s", "fresh-b").longColumn("v", 101).atNow(); - s2.flush(); - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && handler.dictSnapshot().size() < 2) { - Thread.sleep(20); - } - Assert.assertEquals("the new rows must reach the server", - 2, handler.dictSnapshot().size()); - } - - assertUnreplayableSlotSetAside(); - List dict2 = handler.dictSnapshot(); - Assert.assertEquals("the server must hold exactly the post-recovery symbols", - Arrays.asList("fresh-a", "fresh-b"), dict2); - for (String s : dict2) { - Assert.assertFalse("no symbol of the unreplayable slot may reach the server: " + s, - s.startsWith("torn-")); - } - } + assertSlotRecoversWithCompleteDictionary(); }); } @Test - public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { + public void testUnopenableDictRebuildsFromFramesFromIdZero() throws Exception { + // Same rebuild, but the dictionary cannot be OPENED at all (fd exhaustion, a + // read-only remount, ENOSPC -- modelled by planting a directory in its place). The + // producer's seed used to be gated on the dictionary having opened, which made the + // whole rebuild dead code for exactly this case. assertMemoryLeak(() -> { - // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. - // Silent server + close-fast leaves all frames unacked in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } + recordSixDeltaFrames(); + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + writeAckWatermark(slot.resolve(".ack-watermark"), 0); + assertSlotRecoversWithCompleteDictionary(); + }); + } - // Simulate a host/power crash: the segment frames survive but the WHOLE - // persisted dictionary is lost (truncate .symbol-dict to its 8-byte header, - // 0 symbols), and the ack watermark was left mid-stream (FSN 2). The - // surviving frames still reference the lost ids. + @Test + public void testUnopenableDictRebuildsFromFramesAcrossTheAckWatermark() throws Exception { + // The hardest of the three: the dictionary is unopenable AND the replay set starts at + // deltaStart=3, so ids 0..2 exist nowhere except the acked frames on disk. + assertMemoryLeak(() -> { + recordSixDeltaFrames(); java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); - byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); - java.nio.file.Files.write(dict, header); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: recover against a fresh counting server. With the whole - // dictionary lost (size 0) while the surviving frames still reference its - // ids, seedGlobalDictionaryFromPersisted must refuse the resume at BUILD -- - // its size==0 short-circuit must NOT skip the torn-dict guard -- rather than - // let the producer resume unseeded and silently misattribute reused ids. - CountingHandler handler = new CountingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - // The dictionary cannot be rebuilt from any source, so this slot is genuinely - // unreplayable -- and build() SETS IT ASIDE rather than failing. Failing here - // is what bricked the sender: senderId is stable and a not-fully-drained slot - // is retained on close, so every retry re-recovered the same slot and threw - // again, and the application could not construct a Sender at all -- it could - // not even buffer new rows. Now the producer comes up on a clean slot and the - // unreplayable bytes are kept aside for resend. - Sender.fromConfig(cfg).close(); - - // The safety property is unchanged: not one frame of the unreplayable slot - // may reach the server. - Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", - 0, handler.frames.get()); - assertUnreplayableSlotSetAside(); - } + assertSlotRecoversWithCompleteDictionary(); }); } @Test - public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { - // One-id-gap boundary variant of - // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames: the first - // replayed frame starts EXACTLY ONE id past the empty mirror -- the tightest - // gap the I/O-thread replay guard must still reject (deltaStart == - // sentDictCount + 1). It reaches that guard via the unopenable-dictionary path - // (deltaDictEnabled=false), NOT a size-0 openable dict, which now fails clean - // earlier in seedGlobalDictionaryFromPersisted. A one-entry-short tail is the - // common host-crash outcome, so this pins the guard's false-negative edge: a - // "deltaStart > size + 1" mutation would let this single-id gap through and - // null-pad the missing symbol on the server. + public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() throws Exception { + // The genuinely unrecoverable slot -- and the only one left, now that a torn dictionary + // rebuilds from the frames that are still on disk. + // + // Here the frames that REGISTERED the early ids are gone for good: trim unlinked their + // whole segment once they were acked (modelled by deleting sf-initial.sfa, which is + // exactly what SegmentManager does). The dictionary is torn away too, so nothing + // anywhere still holds those ids, and the surviving frames' deltas start above them. + // Replaying would null-pad the hole and silently misattribute symbol values. + // + // So the slot is unreplayable -- and it is SET ASIDE, not allowed to brick the sender. + // Failing here is what bricked it: senderId is stable and a not-fully-drained slot is + // retained on close, so every retry re-recovered the same slot and threw again, and the + // application could not construct a Sender at all -- it could not even buffer new rows. assertMemoryLeak(() -> { - // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i). try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); silent.start(); Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // Small segments so the frames roll into several files and the earliest ones + // can be trimmed away independently. String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; + + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { + for (int i = 0; i < 12; i++) { s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); s1.flush(); } } } - // Make the persisted dictionary UNOPENABLE (replace .symbol-dict with a - // directory, so both openRW and openCleanRW fail and the engine reports - // deltaDictEnabled=false), then stamp the watermark at FSN 0 so recovery - // replays from FSN 1 -- a frame with deltaStart=1. The mirror is unseeded - // (size 0), so id 0 is the single missing entry: deltaStart(1) == - // sentDictCount(0) + 1. No catch-up is sent, so any counted frame would be - // the gapped data frame. java.nio.file.Path slot = Paths.get(sfDir, "default"); + Assert.assertTrue("the frames must have rolled into more than one segment", + countSegmentFiles(slot) > 1); + // TRIM the segment that holds the earliest frames -- munmap + unlink is exactly what + // SegmentManager does once they are acked. Their symbols are now gone from disk. + java.nio.file.Files.delete(slot.resolve("sf-initial.sfa")); + // ...and the dictionary is torn away as well, so nothing holds them at all. java.nio.file.Path dict = slot.resolve(".symbol-dict"); - java.nio.file.Files.delete(dict); - java.nio.file.Files.createDirectory(dict); - writeAckWatermark(slot.resolve(".ack-watermark"), 0); + java.nio.file.Files.write(dict, + Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); - // Phase 2: recover against a fresh counting server. The guard must fire on - // the very first replay frame (deltaStart 1 > recovered size 0) and fail - // terminally rather than send a frame that null-pads id 0. - CountingHandler handler = new CountingHandler(); + DictReconstructingHandler handler = new DictReconstructingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); good.start(); Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); - try { + // build() must SUCCEED -- on a fresh slot -- and the producer must keep working. + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow(); + s2.flush(); long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - s2.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - s2.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { + Thread.sleep(20); } } - Assert.assertEquals("a one-id gap must still block replay to a fresh server", - 0, handler.frames.get()); - Assert.assertNotNull("a one-id-short dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("delta start 1 exceeds recovered dictionary size 0")); + assertUnreplayableSlotSetAside(); + // The producer's new data reaches the server, and not one symbol of the + // unreplayable slot does. + Assert.assertEquals("the producer must keep producing on its fresh slot", + Arrays.asList("after-recovery"), handler.dictSnapshot()); } }); } + private static int countSegmentFiles(java.nio.file.Path dir) { + java.io.File[] files = dir.toFile().listFiles(); + int n = 0; + if (files != null) { + for (java.io.File f : files) { + if (f.getName().endsWith(".sfa")) { + n++; + } + } + } + return n; + } + @Test public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's @@ -516,87 +443,6 @@ public void testUnopenablePersistedDictReplaysSelfSufficientFrameSequence() thro }); } - @Test - public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { - // C1 regression: when a recovered disk slot's persisted dictionary cannot be - // OPENED (fd exhaustion, a read-only remount, ENOSPC -- simulated here by a - // .symbol-dict that is a DIRECTORY, so both openRW and openCleanRW fail), - // CursorSendEngine.isDeltaDictEnabled() returns false. The recorded frames - // are still DELTA frames, and replaying them against a fresh - // empty-dictionary server would null-pad the missing ids and SILENTLY - // corrupt the table. The torn-dictionary guard must fire regardless of - // deltaDictEnabled -- pre-fix it was gated on that very flag, so the - // corrupting frame sailed through unguarded. Unlike - // testTornDictionaryFailsCleanlyInsteadOfCorrupting (dict present but empty, - // deltaDictEnabled=true), here the dict is UNOPENABLE (deltaDictEnabled=false). - assertMemoryLeak(() -> { - // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } - - // Make the persisted dictionary UNOPENABLE: replace the .symbol-dict file - // with a directory of the same name so PersistedSymbolDict.open() returns - // null (both openRW and openCleanRW fail) and the engine reports - // deltaDictEnabled=false. Stamp the watermark at FSN 2 so replay starts - // at FSN 3 -- a frame whose delta starts at id 3, with ids 0..2 living - // only in the now-unreadable dictionary. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - java.nio.file.Path dict = slot.resolve(".symbol-dict"); - java.nio.file.Files.delete(dict); - java.nio.file.Files.createDirectory(dict); - writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: recover against a fresh counting server. The guard must fire - // (frame deltaStart 3 > recovered dictionary size 0) and fail terminally - // rather than send a gapped frame that would corrupt the table. - CountingHandler handler = new CountingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); - try { - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - s2.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - s2.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } - } - } - Assert.assertEquals("no delta frame may be replayed when the persisted dictionary is unopenable", - 0, handler.frames.get()); - Assert.assertNotNull("an unopenable dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("symbol dictionary is incomplete")); - } - }); - } - @Test public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Exception { // C3 regression: sendCommitMessage does NOT write-ahead-persist the @@ -1428,4 +1274,58 @@ private static boolean hasSegmentFile(java.nio.file.Path dir) { return false; } + + /** + * Phase 1 for the recovery tests: six frames, each introducing exactly one new symbol + * (sym-0 .. sym-5), left unacked on disk by a silent server and a fast close. + */ + private void recordSixDeltaFrames() throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + } + + /** + * Recovers the slot against a fresh server and asserts the dictionary it ends up holding + * is COMPLETE and IN ORDER. + *

    + * This is the strong form, and it is the point: the fresh server starts with an empty + * dictionary, so ids the replayed frames' deltas start ABOVE can only come from a catch-up + * frame -- which can only carry them if the mirror was seeded from the frames still on + * disk. A dictionary that came back null-padded (the server's response to an id it has + * never seen) or shifted by one would fail here, and that is precisely the corruption the + * old guard condemned these slots to avoid. It never had to. + */ + private void assertSlotRecoversWithCompleteDictionary() throws Exception { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 6) { + Thread.sleep(20); + } + s2.flush(); + } + Assert.assertEquals( + "every id is still held by a frame on disk, so the catch-up must rebuild the " + + "dictionary COMPLETE and gap-free -- not null-pad it", + Arrays.asList("sym-0", "sym-1", "sym-2", "sym-3", "sym-4", "sym-5"), + handler.dictSnapshot()); + } + } + } From 3b73bc815d0c560f9aec93a6e739b4dbbd53b77f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 21:08:12 +0100 Subject: [PATCH 60/78] Pin the producer's recovery seed on the unopenable-dictionary path The seed's ungating had no test: nothing wrote a NEW symbol after recovering a slot whose dictionary could not be opened, which is the only place the difference shows. It shows on the wire. Re-gate the seed and the resumed producer restarts its id space at 0, on top of ids the surviving frames already define -- the server's dictionary comes back as [after-recovery, sym-1, sym-2, sym-3, sym-4, sym-5] with sym-0 overwritten. Two symbols on one id, and the send loop's mirror still reads that id as sym-0, so mirror and producer disagree about what it MEANS. That is the misattribution the whole dictionary machinery exists to prevent. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictRecoveryTest.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index aaaeb66a..e4c36c30 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -287,6 +287,57 @@ private static int countSegmentFiles(java.nio.file.Path dir) { return n; } + @Test + public void testUnopenableDictSeedsTheProducerAboveTheRecoveredIds() throws Exception { + // The producer must resume ABOVE the ids the recovered frames already define, even when + // the dictionary could not be opened. + // + // seedGlobalDictionaryFromPersisted used to be gated on deltaDictEnabled, which is false + // exactly here -- so the producer restarted its id space at 0, on top of ids the + // surviving frames define. The send loop's mirror (rebuilt from those frames) still read + // id 0 as sym-0 while the producer meant something else by it: the two disagree about + // what an id MEANS, which is the whole failure mode the dictionary machinery exists to + // prevent. + // + // Visible on the wire: the new symbol must land ABOVE the recovered ones, not on top of + // sym-0. + assertMemoryLeak(() -> { + recordSixDeltaFrames(); + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 6) { + Thread.sleep(20); + } + // ...and now the resumed producer introduces a symbol of its own. + s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow(); + s2.flush(); + deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 7) { + Thread.sleep(20); + } + } + Assert.assertEquals( + "the resumed producer must take the NEXT id, not reuse id 0 -- reusing it " + + "puts two symbols on one id and silently misattributes values", + Arrays.asList("sym-0", "sym-1", "sym-2", "sym-3", "sym-4", "sym-5", + "after-recovery"), + handler.dictSnapshot()); + } + }); + } + @Test public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's From 98a4ac033db47aac1e422686b3b225fae17341cd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 23:32:11 +0100 Subject: [PATCH 61/78] Don't key the cap-gap episode off nanoTime's sign sendDictCatchUp tested `catchUpCapGapFirstNanos < 0` to decide whether a symbol-dictionary cap-gap episode was already open. A System.nanoTime() instant is only meaningful as a difference: its origin is arbitrary and the spec permits negative values, so its sign cannot encode state. CursorWebSocketSendLoopCatchUpAlignmentTest backdates the episode anchor by two hours to prove that a cap gap outliving the escalation window latches a terminal. On Linux nanoTime() is nanos-since-boot, so on a freshly booted CI agent it sits far below two hours of nanos and the backdated anchor came out negative. The loop then read that real anchor as "unset", re-anchored it to now on every strike and pinned episodeNanos at ~0: the dwell was never satisfied, no terminal latched, and the test failed. It fails on every machine with under two hours of uptime and on no long-lived dev box, which is why only CI saw it. Key the episode off the strike count instead, exactly as recordHeadRejectionStrike keys its own off poisonStrikes. -1 survives as a debug marker but no longer decides anything, so no state rides on a timestamp's sign. Add testCapGapEpisodeWithANegativeAnchorStillEscalates, which plants the negative anchor directly and so pins the sentinel whatever the host's uptime; the existing test only reddens where uptime happens to fall under its two-hour backdate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 34 +++++++--- ...WebSocketSendLoopCatchUpAlignmentTest.java | 65 +++++++++++++++++++ 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index cf3f1637..9d198fff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -301,15 +301,19 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // nor resets it. NOT reset per connection -- it measures the cap-gap episode // across reconnects so a persistent gap eventually latches. I/O-thread-only. private int catchUpCapGapAttempts; - // System.nanoTime() of the FIRST cap gap of the current episode, or -1 when no - // episode is open. Anchors the escalation dwell. Reset together with - // catchUpCapGapAttempts on a successful catch-up, so a node that accepts the - // dictionary ends the episode outright. Anchoring at the FIRST gap (not at loop - // entry, and not refreshed per attempt) is what keeps a TRANSIENT from burning the - // terminal budget: a transient never reaches the catch-up, so it neither increments - // the counter nor moves the anchor -- it can only lengthen the wall clock, which - // alone can never latch the terminal because the strike count still has to be - // satisfied. I/O-thread-only, like catchUpCapGapAttempts. + // System.nanoTime() of the FIRST cap gap of the current episode. Anchors the + // escalation dwell. Reset together with catchUpCapGapAttempts on a successful + // catch-up, so a node that accepts the dictionary ends the episode outright. + // Anchoring at the FIRST gap (not at loop entry, and not refreshed per attempt) is + // what keeps a TRANSIENT from burning the terminal budget: a transient never reaches + // the catch-up, so it neither increments the counter nor moves the anchor -- it can + // only lengthen the wall clock, which alone can never latch the terminal because the + // strike count still has to be satisfied. I/O-thread-only, like catchUpCapGapAttempts. + // + // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test + // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant + // is only meaningful as a difference: its origin is arbitrary and the spec permits + // negative values, so no state may ride on this field's sign. private long catchUpCapGapFirstNanos = -1L; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up consumes @@ -2442,8 +2446,18 @@ private int sendDictCatchUp() { // an ordinary rolling restart of the larger-cap node. Escalating on the // count alone would therefore hard-fail a live store-and-forward // producer during a routine cluster operation. + // + // The STRIKE COUNT -- never the anchor's sign -- says whether an episode is + // already open. A System.nanoTime() instant is only meaningful as a + // difference: its origin is arbitrary and the spec permits negative values, + // so a sign test cannot tell an unset anchor from a real one. Wherever it + // misreads a real anchor as unset it re-anchors to now on EVERY strike, + // pinning episodeNanos at ~0, and the dwell can then never be satisfied -- + // the terminal never latches, however long the gap truly persists. + // recordHeadRejectionStrike keys its episode off poisonStrikes for the same + // reason. long nowNanos = System.nanoTime(); - if (catchUpCapGapFirstNanos < 0) { + if (catchUpCapGapAttempts == 0) { catchUpCapGapFirstNanos = nowNanos; } catchUpCapGapAttempts++; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 707ef5a8..4e00e1e6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -330,6 +330,71 @@ public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() t }); } + @Test + public void testCapGapEpisodeWithANegativeAnchorStillEscalates() throws Exception { + // A cap-gap episode anchored at a NEGATIVE nanoTime instant must escalate like any + // other. A System.nanoTime() value is only meaningful as a difference -- its origin + // is arbitrary and the spec permits negative values -- so no state may ride on the + // anchor's sign. sendDictCatchUp once tested catchUpCapGapFirstNanos < 0 to mean "no + // episode open": that read a negative anchor as unset, re-anchored it to now on every + // strike and pinned episodeNanos at ~0, so the dwell was never satisfied and the + // terminal could never latch, however long the cap gap truly persisted. + // + // That is what reddened CI. The sibling test above backdates the anchor two hours, + // and on Linux nanoTime() is nanos-since-boot: on a CI agent up ten minutes it is + // ~6e11, so "two hours ago" comes out ~ -6.6e12 -- negative. The defect therefore + // only surfaced where uptime is under that backdate: every fresh CI agent, and never + // a long-lived dev box (which is why it passed locally). Planting the negative anchor + // directly pins the sentinel on ANY machine, whatever its uptime. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + // A one-hour dwell, against an anchor two hours back: satisfied on elapsed, + // but only if the negative anchor survives to the subtraction. + CursorWebSocketSendLoop loop = newLoop(engine, client, 3_600_000L); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + // Satisfy the strike half of the AND, one short of the budget. + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); // dwell unmet => retriable, whatever the count + } + // The episode began two hours ago on a machine booted minutes ago. + Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos"); + anchor.setAccessible(true); + anchor.setLong(loop, -TimeUnit.HOURS.toNanos(2)); + + // Both halves now hold, so this strike must latch the terminal. + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("the escalating cap gap must still raise CatchUpSendException"); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + try { + loop.checkError(); + fail("a cap-gap episode anchored at a negative nanoTime instant must still " + + "escalate -- the anchor's sign carries no meaning"); + } catch (LineSenderException terminal) { + assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up") + && terminal.getMessage().contains("must be resent")); + } + } finally { + loop.close(); + } + } + }); + } + @Test public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Exception { // A TRANSIENT catch-up failure (the wire drops mid-catch-up -- a flapping LB, a From 63dc52a55b096e507b42a12ec9efe0f394e4ae91 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 01:30:19 +0100 Subject: [PATCH 62/78] Free the recovery mirror on a ctor seed throw The CursorWebSocketSendLoop constructor mallocs the sent-dictionary mirror from the persisted dictionary's prefix, then extends it from the surviving frames' deltas. If that second step throws -- a native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling -- the throw leaves the loop unpublished, so neither ensureConnected's catch nor BackgroundDrainer's finally can close() it, and the already-malloc'd mirror leaks. Free it on any throw, mirroring the loopNeverRan free in close()/ioLoop's exit. Also close two test-coverage gaps the review flagged: - The send loop's torn-dict guard (deltaStart > sentDictCount in trySendOne) is the drainer path's sole defense against replaying a frame whose symbols were trimmed away; the foreground path is quarantined earlier, so this fire direction was unexercised. Cover it at the send-loop level against a stub client (the drainer integration path cannot connect on darwin), asserting zero frames ship and the terminal latches. - quarantineTornSlot must fail loudly rather than drop bytes when it cannot set a slot aside. Saturate every quarantine candidate and assert build() throws and preserves the slot on disk. A TestOnly seam (forceMirrorSeedFailureForTest) makes the ctor seed throw deterministically; the unreplayable-slot setup is now a shared helper. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 39 +++- .../qwp/client/DeltaDictRecoveryTest.java | 103 +++++--- ...CursorWebSocketSendLoopMirrorLeakTest.java | 89 +++++++ ...sorWebSocketSendLoopTornDictGuardTest.java | 220 ++++++++++++++++++ 4 files changed, 421 insertions(+), 30 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 9d198fff..c705aaac 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -197,6 +197,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // Test seam: when true, appendSymbolToMirror throws instead of appending, + // simulating the native realloc OOM (or MAX_SENT_DICT_BYTES ceiling) that + // ensureSentDictCapacity can raise while the constructor seeds the recovery + // mirror. Lets a test prove the constructor frees the already-malloc'd mirror on + // such a throw rather than leaking it. Production never sets it; volatile only so + // a test thread's write is visible to the loop under test. + @TestOnly + static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -717,11 +725,30 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // nothing and cost a catch-up frame on every connection -- full-dict slots must stay // catch-up-free. if (engine.recoveredMaxSymbolDeltaStart() > 0L) { - ObjList fromFrames = new ObjList<>(); - if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { - for (int i = 0, n = fromFrames.size(); i < n; i++) { - appendSymbolToMirror(fromFrames.getQuick(i)); + // The prefix seed above may already have malloc'd the mirror, and + // appendSymbolToMirror -> ensureSentDictCapacity can still throw here (a + // native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling). Such a throw + // propagates out of the constructor, so the half-built loop is never + // assigned to a reference -- neither ensureConnected's catch nor + // BackgroundDrainer's finally can close() it -- and the mirror would leak. + // Free it on any throw so the constructor leaves nothing behind, mirroring + // the loopNeverRan free in close() / ioLoop's exit. + try { + ObjList fromFrames = new ObjList<>(); + if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { + for (int i = 0, n = fromFrames.size(); i < n; i++) { + appendSymbolToMirror(fromFrames.getQuick(i)); + } } + } catch (Throwable t) { + if (sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + sentDictCount = 0; + } + throw t; } } this.fsnAtZero = fsnAtZero; @@ -2299,6 +2326,10 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart * construction; {@link #accumulateSentDict} extends it from live frames thereafter. */ private void appendSymbolToMirror(CharSequence symbol) { + if (forceMirrorSeedFailureForTest) { + // Simulate a native realloc OOM on the seed path (see the field). + throw new LineSenderException("simulated mirror seed allocation failure (test only)"); + } int utf8Len = Utf8s.utf8Bytes(symbol); int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; ensureSentDictCapacity((long) sentDictBytesLen + wireLen); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e4c36c30..e8ff1e99 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -223,32 +223,7 @@ public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() th // retained on close, so every retry re-recovered the same slot and threw again, and the // application could not construct a Sender at all -- it could not even buffer new rows. assertMemoryLeak(() -> { - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - // Small segments so the frames roll into several files and the earliest ones - // can be trimmed away independently. - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 12; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } - - java.nio.file.Path slot = Paths.get(sfDir, "default"); - Assert.assertTrue("the frames must have rolled into more than one segment", - countSegmentFiles(slot) > 1); - // TRIM the segment that holds the earliest frames -- munmap + unlink is exactly what - // SegmentManager does once they are acked. Their symbols are now gone from disk. - java.nio.file.Files.delete(slot.resolve("sf-initial.sfa")); - // ...and the dictionary is torn away as well, so nothing holds them at all. - java.nio.file.Path dict = slot.resolve(".symbol-dict"); - java.nio.file.Files.write(dict, - Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); + writeAndTearUnreplayableSlot(); DictReconstructingHandler handler = new DictReconstructingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { @@ -287,6 +262,82 @@ private static int countSegmentFiles(java.nio.file.Path dir) { return n; } + @Test + public void testQuarantineFailsLoudlyWhenAllSlotNamesSaturated() throws Exception { + // M2 regression: when a recovered slot is genuinely unreplayable, build() sets + // it aside (quarantineTornSlot) and starts the producer on a fresh slot. But if + // it cannot free the slot name -- here every default.unreplayable- candidate + // up to MAX_QUARANTINE_SLOT_ATTEMPTS (64) already exists -- it MUST fail LOUDLY: + // throw and leave the slot's bytes on disk for a manual resend, never silently + // drop data. Only the happy rename path was covered before. + assertMemoryLeak(() -> { + writeAndTearUnreplayableSlot(); + // Saturate every quarantine candidate so quarantineTornSlot's rename loop + // finds no free name. + for (int i = 0; i < 64; i++) { + java.nio.file.Files.createDirectories(Paths.get(sfDir, "default.unreplayable-" + i)); + } + + java.nio.file.Path slot = Paths.get(sfDir, "default"); + try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + Sender s = null; + try { + s = Sender.fromConfig(cfg); + Assert.fail("build() must throw when the unreplayable slot cannot be set aside"); + } catch (LineSenderException expected) { + Assert.assertTrue("unexpected message: " + expected.getMessage(), + expected.getMessage().contains("too many quarantined slots already under") + && expected.getMessage().contains("moved or removed by hand")); + } finally { + if (s != null) { + s.close(); + } + } + } + // The unreplayable slot's bytes must survive on disk for a manual resend -- + // the guard fails loudly rather than dropping data. + Assert.assertTrue("the slot dir must be preserved", java.nio.file.Files.exists(slot)); + Assert.assertTrue("the slot's segment data must be preserved", + countSegmentFiles(slot) >= 1); + }); + } + + // Writes 12 delta frames (each introducing a new symbol) into the default slot + // across several small segments, then makes the slot GENUINELY unreplayable: trims + // the segment holding the earliest ids (munmap + unlink, exactly what SegmentManager + // does once they are acked) and tears the .symbol-dict down to its header, so the + // surviving frames' deltas start above ids nothing on disk still holds. Recovering + // such a slot throws UnreplayableSlotException. + private void writeAndTearUnreplayableSlot() throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // Small segments so the frames roll into several files and the earliest ones + // can be trimmed away independently. + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 12; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + java.nio.file.Path slot = Paths.get(sfDir, "default"); + Assert.assertTrue("the frames must have rolled into more than one segment", + countSegmentFiles(slot) > 1); + // TRIM the segment that holds the earliest frames. + java.nio.file.Files.delete(slot.resolve("sf-initial.sfa")); + // ...and tear the dictionary away as well, so nothing holds those ids at all. + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.write(dict, Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); + } + @Test public void testUnopenableDictSeedsTheProducerAboveTheRecoveredIds() throws Exception { // The producer must resume ABOVE the ids the recovered frames already define, even when diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 5cff81ba..8697eaee 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; @@ -156,6 +157,63 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { } } + @Test + public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { + // C1 regression: the constructor seeds the recovery mirror in TWO steps -- a + // malloc from the persisted dictionary's intact prefix, then an extension from + // the surviving frames' own deltas (appendSymbolToMirror). If that second step + // throws (a native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling), the throw + // leaves the constructor with the object unpublished, so neither + // ensureConnected's catch nor BackgroundDrainer's finally can ever close() it -- + // and the already-malloc'd prefix mirror leaks. The constructor must free it on + // the throw. Pre-fix, the mirror leaks here and assertMemoryLeak fails. + Path sfDir = Files.createTempDirectory("qwp-mirror-ctor-throw"); + try { + // A torn-dict SUBSET: three delta frames a@0,b@1,c@2 survive on disk, but the + // .symbol-dict is rewritten to hold only [a,b] (a host-crash tail tear). On + // recovery pd.size()==2 seeds (mallocs) the mirror, then the frame-seed + // rebuilds c@2 from the surviving frame -- the append the fault interrupts. + populateThreeFrameSlot(sfDir); + Path slot = sfDir.resolve("default"); + try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slot.toString())) { + Assert.assertNotNull(torn); + torn.appendSymbol("a"); + torn.appendSymbol("b"); + Assert.assertEquals(2, torn.size()); + } + + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull("recovery must open the torn subset dict", pd); + Assert.assertEquals("prefix seed must malloc a 2-entry mirror", 2, pd.size()); + Assert.assertTrue("the frame-seed path must run (frames out-reach the dict)", + engine.recoveredMaxSymbolDeltaStart() > 0L); + + setMirrorSeedFault(true); + try { + new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + Assert.fail("ctor must propagate the injected mirror-seed failure"); + } catch (LineSenderException expected) { + Assert.assertTrue("unexpected message: " + expected.getMessage(), + expected.getMessage().contains("simulated mirror seed allocation failure")); + } finally { + setMirrorSeedFault(false); + } + // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor + // malloc'd was freed on the throw -- pre-fix it leaks here. + } + }); + } finally { + rmDir(sfDir); + } + } + // Constructs a recovery send loop but does NOT start it: the ctor seeds the // catch-up mirror synchronously, which is all these tests observe. The // reconnect factory is never invoked. @@ -189,12 +247,43 @@ private static void populateRecoverableSlot(Path sfDir) throws Exception { } } + // Three delta frames a@0, b@1, c@2, nothing acked, so all three survive and + // replay from frame 0. Paired with a dictionary truncated to [a,b], this is a + // torn-dict SUBSET whose recovery drives the constructor's frame-seed path. + private static void populateThreeFrameSlot(Path sfDir) throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s = Sender.fromConfig(cfg)) { + s.table("m").symbol("s", "a").longColumn("v", 0).atNow(); + s.flush(); + s.table("m").symbol("s", "b").longColumn("v", 1).atNow(); + s.flush(); + s.table("m").symbol("s", "c").longColumn("v", 2).atNow(); + s.flush(); + } + } + } + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); f.setAccessible(true); return f.getInt(loop); } + // Toggles the loop's @TestOnly mirror-seed fault flag. Reflection because the + // flag is package-private in the production package (this test is in a sibling + // test package), the same non-reflective-path-unavailable reason readInt uses. + private static void setMirrorSeedFault(boolean value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField("forceMirrorSeedFailureForTest"); + f.setAccessible(true); + f.setBoolean(null, value); + } + private static void rmDir(Path dir) throws IOException { TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java new file mode 100644 index 00000000..7dabc741 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java @@ -0,0 +1,220 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * White-box coverage of the send loop's torn-dictionary guard + * ({@code trySendOne}: {@code deltaStart > sentDictCount}) firing on a genuine gap. + *

    + * That guard is the drainer path's SOLE defense: an orphan drainer adopts a slot + * without running {@code Sender.build()}'s seed-time guard, so on a slot whose + * registering frames were trimmed away and whose {@code .symbol-dict} was torn, the + * send loop must detect the gap itself, ship ZERO frames, and latch a terminal -- + * never null-pad the hole on the server. The foreground sender is always quarantined + * earlier (at build time), so this fire direction runs only here. Driven at the send + * loop level against a frame-counting stub client so it exercises the guard + * deterministically, without a real network connection. + */ +public class CursorWebSocketSendLoopTornDictGuardTest { + + private String sfDir; + + @Before + public void setUp() { + sfDir = TestUtils.createTmpDir("qdb-torn-dict-guard-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(sfDir); + } + + @Test + public void testGuardFiresOnGenuineGapAndShipsNoFrame() throws Exception { + assertMemoryLeak(() -> { + writeAndTearGappedSlot(); + + CountingClient client = new CountingClient(); + try (CursorSendEngine engine = new CursorSendEngine(sfDir + "/default", 16_384)) { + // The persisted dict was torn away and the registering frames trimmed, so + // the mirror cannot be seeded from either source: sentDictCount stays 0 + // while the first surviving frame's delta starts above it. + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + setRunning(loop); + // Position at the first unsent frame exactly as the I/O loop does + // before its first send (no catch-up: the mirror is empty). + invoke(loop, "positionCursorForStart"); + + boolean sent = (Boolean) invoke(loop, "trySendOne"); + + Assert.assertFalse("the torn-dict guard must refuse to send the gapped frame", sent); + Assert.assertEquals("no frame may reach the server through a gap", + 0, client.framesSent); + try { + loop.checkError(); + Assert.fail("the guard must latch a terminal error"); + } catch (LineSenderException e) { + Assert.assertTrue("unexpected terminal: " + e.getMessage(), + e.getMessage().contains("incomplete") + && e.getMessage().contains("resend required")); + } + } finally { + loop.close(); + } + } + }); + } + + private static int countSegmentFiles(Path dir) { + File[] files = dir.toFile().listFiles(); + int n = 0; + if (files != null) { + for (File f : files) { + if (f.getName().endsWith(".sfa")) { + n++; + } + } + } + return n; + } + + private static Object invoke(CursorWebSocketSendLoop loop, String method) throws Exception { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(method); + m.setAccessible(true); + return m.invoke(loop); + } + + private static void setRunning(CursorWebSocketSendLoop loop) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField("running"); + f.setAccessible(true); + f.setBoolean(loop, true); + } + + // Constructs a recovery send loop that is never started -- the test drives + // positionCursorForStart + trySendOne directly. The reconnect factory throws + // because no reconnect is expected before the guard latches its terminal. + private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { + return new CursorWebSocketSendLoop( + client, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + } + + // Writes 12 delta frames (each a new symbol) into the default slot across several + // small segments, then makes the slot carry a GENUINE gap: trims the segment + // holding the earliest ids (munmap + unlink, as SegmentManager does once they are + // acked) and tears the .symbol-dict down to its header. The surviving frames' deltas + // then start above ids nothing on disk still holds, so the mirror cannot be rebuilt. + private void writeAndTearGappedSlot() throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; + try (Sender s = Sender.fromConfig(cfg)) { + for (int i = 0; i < 12; i++) { + s.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s.flush(); + } + } + } + Path slot = Paths.get(sfDir, "default"); + Assert.assertTrue("the frames must have rolled into more than one segment", + countSegmentFiles(slot) > 1); + Files.delete(slot.resolve("sf-initial.sfa")); + Path dict = slot.resolve(".symbol-dict"); + Files.write(dict, Arrays.copyOf(Files.readAllBytes(dict), 8)); + } + + // Frame-counting stub transport: completes no real I/O. If the guard ever lets a + // gapped frame through, sendBinary bumps framesSent and the test fails. + private static final class CountingClient extends WebSocketClient { + private int framesSent; + + CountingClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public int getServerMaxBatchSize() { + return 16_384; + } + + @Override + public int getServerQwpVersion() { + return 1; + } + + @Override + public void sendBinary(long dataPtr, int length) { + framesSent++; + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- the sender leaves everything unacked in the slot + } + } +} From 7299925df7c5143e5a822d3dc36aa4359a730edd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 01:47:39 +0100 Subject: [PATCH 63/78] Correct stale and stranded symbol-dict comments Comment-only follow-ups from the delta symbol-dict review; no behaviour change. - quarantineTornSlot marks the set-aside slot .failed unconditionally, so the orphan drainer never re-examines it. Fix the two comments that claimed the drainer could still deliver a quarantined slot; state that it is a human-in-the-loop copy whose bytes are kept for a manual resend. - sendCommitMessage's full-dict-mode comment claimed the commit frame stays self-sufficient, but the prior flush reset currentBatchMaxSymbolId to -1, so the frame carries an empty delta. Say so -- it is harmless, the preceding data frames already registered the dictionary. - Move three javadoc blocks stranded above the wrong method back onto their own: MmapSegment.maxSymbolDeltaEnd, SegmentRing.maxSymbolDeltaEnd and Sender.closeFlushTimeoutMillis were left undocumented, and the stranded @param names no longer matched the method beneath them. - Drop a double blank line in CursorSendEngine. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../main/java/io/questdb/client/Sender.java | 57 +++++++++-------- .../qwp/client/QwpWebSocketSender.java | 4 +- .../client/sf/cursor/CursorSendEngine.java | 1 - .../qwp/client/sf/cursor/MmapSegment.java | 62 +++++++++---------- .../qwp/client/sf/cursor/SegmentRing.java | 16 ++--- 5 files changed, 73 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index b674cf1b..23fdf6a8 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1013,8 +1013,9 @@ final class LineSenderBuilder { private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the - // sender's own slot name, so OrphanScanner sees the quarantined copy and the - // orphan drainer can still deliver it if its frames turn out to be replayable. + // sender's own slot name, so a restarted sender does not re-adopt it as its own; + // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and + // the bytes stay put for a human to inspect and resend. private static final String QUARANTINE_SLOT_SUFFIX = ".unreplayable-"; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); @@ -1757,23 +1758,6 @@ public Sender build() { return sender; } - /** - * close() drain timeout in milliseconds. The sender's {@code close()} - * method blocks up to this many millis waiting for the server to ACK - * every batch already published into the engine before shutting down - * the I/O loop. Default {@code 60000} (60 s) -- generous enough to - * survive real-workload backlogs (slow consumers, catch-up replicas, - * chunky payloads on small server send buffers) without silently - * dropping unacked rows; callers that need a longer pre-close wait - * for a specific submission can call - * {@link Sender#drain(long)} explicitly before close(). - *

    - * Set to {@code 0} or {@code -1} to opt out — close() will not wait - * at all (fast close). Pending data is then lost in memory mode and - * recovered by the next sender in SF mode. - *

    - * WebSocket transport only. - */ /** * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must * persist before the sender gives up on it and fails. @@ -1805,6 +1789,23 @@ public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { return this; } + /** + * close() drain timeout in milliseconds. The sender's {@code close()} + * method blocks up to this many millis waiting for the server to ACK + * every batch already published into the engine before shutting down + * the I/O loop. Default {@code 60000} (60 s) -- generous enough to + * survive real-workload backlogs (slow consumers, catch-up replicas, + * chunky payloads on small server send buffers) without silently + * dropping unacked rows; callers that need a longer pre-close wait + * for a specific submission can call + * {@link Sender#drain(long)} explicitly before close(). + *

    + * Set to {@code 0} or {@code -1} to opt out — close() will not wait + * at all (fast close). Pending data is then lost in memory mode and + * recovered by the next sender in SF mode. + *

    + * WebSocket transport only. + */ public LineSenderBuilder closeFlushTimeoutMillis(long timeoutMillis) { if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("close_flush_timeout_millis is only supported for WebSocket transport"); @@ -3019,13 +3020,17 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na * already-lost batch for an unbounded outage of everything after it, which inverts * the one guarantee store-and-forward exists to give. *

    - * So: rename the slot aside instead. The bytes are preserved for forensics and for - * the orphan drainer, which reaches the same verdict independently (its send loop's - * replay guard fires and it marks the slot {@code .failed}) -- and which, on a slot - * that turns out to be drainable after all (frames written in full-dictionary - * fallback mode are self-sufficient), simply drains it. The new name is NOT the - * sender's own slot name, so {@code OrphanScanner} will consider it. The producer, - * meanwhile, starts on a clean empty slot and never notices. + * So: rename the slot aside instead, and mark it {@code .failed}. The verdict is + * authoritative -- the recovery seed already tried every source of truth (the + * persisted prefix AND the surviving frames' own deltas), and the orphan drainer's + * own replay guard uses that same walk, so there is nothing a drainer could rebuild + * that the seed did not. {@code markFailed} (below) therefore quarantines the copy + * for a human rather than leaving a drainer to retry an unreplayable slot forever; + * a full-dictionary-fallback slot never reaches here, because its dictionary is + * discarded at recovery and it never throws. The bytes are preserved on disk for + * forensics and a manual resend, and the new name -- NOT the sender's own slot name + * -- keeps a restarted sender from re-adopting it. The producer, meanwhile, starts + * on a clean empty slot and never notices. *

    * If the rename fails (a Windows share lock, a read-only mount) there is no way to * free the slot name without destroying data, so fall back to the old behaviour and diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d7ba19cf..afdb8114 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3751,7 +3751,9 @@ private void sendCommitMessage() { // what has already been sent -- and therefore already persisted. In delta // mode pass sentMaxSymbolId, yielding an empty delta // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep - // currentBatchMaxSymbolId so the frame stays self-sufficient. Any symbol a + // currentBatchMaxSymbolId (reset to -1 by the prior flush, so the commit frame + // carries an empty delta too -- harmless, since the preceding full-dict data + // frames already registered the dictionary on this connection). Any symbol a // cancelled row leaked is picked up (and persisted) by the next real flush, // whose persistNewSymbolsBeforePublish resumes from pd.size(). int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 37adf4a0..6b1d741a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -783,7 +783,6 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } - /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 5ad281e1..b2ee674b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -524,37 +524,6 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in return best; } - /** - * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any - * delta-flagged frame in this segment carries, or {@code 0} only when NO - * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit - * or a symbol-reusing frame -- the encoder always sets the delta flag) still - * contributes its {@code deltaStart}, i.e. the producer's baseline at encode - * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that - * baseline so a dictionary torn below it is detected -- a conservative - * over-strand (it may over-reject a frame whose rows reference only surviving - * ids), never an under-strand that could silently shift the dense id map. - * Read-only walk over the recovered frames, used - * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, - * out-of-order page loss) below the ids the surviving frames still reference: if - * the max here reaches at or beyond the recovered dictionary size, a resuming - * producer -- seeded from the shorter dictionary -- would re-use ids the frames - * already define. Only frames at or below {@code maxFsnInclusive} count: frames - * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires - * without ever transmitting, so their ids must not inflate the result (a resuming - * producer never reuses them on the wire). The frame layout mirrors - * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header - * ({@code qwpHeaderSize} bytes) is followed by the delta section - * {@code [deltaStart varint][deltaCount varint]...}. - * - * @param headerMagic the QWP message magic identifying a well-formed frame - * @param flagsOffset byte offset of the flags field within the QWP header - * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit - * @param qwpHeaderSize the QWP message header size (delta section starts past it) - * @param maxFsnInclusive highest frame FSN to consider; frames above it are the - * retired orphan-deferred tail -- pass - * {@code recoveredCommitBoundaryFsn} - */ /** * Rebuilds, from the frames' own delta sections, the symbols a replay of * {@code [minFsnInclusive .. maxFsnInclusive]} would register ABOVE {@code coverage}, @@ -670,6 +639,37 @@ public long collectReplaySymbolsAbove( return coverage; } + /** + * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any + * delta-flagged frame in this segment carries, or {@code 0} only when NO + * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit + * or a symbol-reusing frame -- the encoder always sets the delta flag) still + * contributes its {@code deltaStart}, i.e. the producer's baseline at encode + * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that + * baseline so a dictionary torn below it is detected -- a conservative + * over-strand (it may over-reject a frame whose rows reference only surviving + * ids), never an under-strand that could silently shift the dense id map. + * Read-only walk over the recovered frames, used + * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, + * out-of-order page loss) below the ids the surviving frames still reference: if + * the max here reaches at or beyond the recovered dictionary size, a resuming + * producer -- seeded from the shorter dictionary -- would re-use ids the frames + * already define. Only frames at or below {@code maxFsnInclusive} count: frames + * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires + * without ever transmitting, so their ids must not inflate the result (a resuming + * producer never reuses them on the wire). The frame layout mirrors + * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header + * ({@code qwpHeaderSize} bytes) is followed by the delta section + * {@code [deltaStart varint][deltaCount varint]...}. + * + * @param headerMagic the QWP message magic identifying a well-formed frame + * @param flagsOffset byte offset of the flags field within the QWP header + * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit + * @param qwpHeaderSize the QWP message header size (delta section starts past it) + * @param maxFsnInclusive highest frame FSN to consider; frames above it are the + * retired orphan-deferred tail -- pass + * {@code recoveredCommitBoundaryFsn} + */ public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; long off = HEADER_SIZE; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 3ec06014..06d1e64d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -546,14 +546,6 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag return Math.max(best, fsn); } - /** - * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below - * {@code maxFsnInclusive} references (0 when none). See - * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a - * persisted dictionary torn below the ids the surviving committed frames - * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred - * tail and are excluded. - */ /** * Rebuilds the symbols a replay of {@code [minFsnInclusive .. maxFsnInclusive]} would * register ABOVE {@code baseline}, appending them to {@code out} in ascending id order. @@ -589,6 +581,14 @@ public synchronized long collectReplaySymbolsAbove( minFsnInclusive, maxFsnInclusive, coverage, out); } + /** + * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below + * {@code maxFsnInclusive} references (0 when none). See + * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a + * persisted dictionary torn below the ids the surviving committed frames + * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred + * tail and are excluded. + */ public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; for (int i = 0, n = sealedSegments.size(); i < n; i++) { From c0c2ce272d4fed327dbc10532a5355ab5b416795 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 11:32:21 +0100 Subject: [PATCH 64/78] Preserve symbol dict on a transient stat fault PersistedSymbolDict.open() computed the routing length as ff.length(path) and sent anything below HEADER_SIZE to openFresh(), which O_TRUNCs the file. A stat that fails transiently (an EIO on a flaky disk) returns -1 for a fully populated dictionary, so that routing could truncate the only copy of load-bearing recovery state -- the exact destruction the class's "never recreate over an existing file" contract forbids, and which the openExisting read path already guards against but the routing check did not. A stat error reports -1, distinguishable from a genuine sub-header stub in [0, HEADER_SIZE), so open() now degrades to null when the file exists but its length reads negative: it falls back to full self-sufficient frames and leaves every byte on disk for a later attempt, once the transient clears. Also move appendRawEntries' per-entry (addr, len, count) consistency walk behind an assert. It re-reads every entry's length prefix on the common flush path, yet its sole caller derives count and len from one beginMessage, so it cannot fire today. The assert keeps the structural guard in the client's -ea test suite while eliding it from production, where client apps run without -ea. PersistedSymbolDictTest now covers a transient length fault leaving the file intact and recovering once it clears, and an inconsistent raw-entries triple being rejected under -ea. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 95 +++++++++----- .../sf/cursor/PersistedSymbolDictTest.java | 118 ++++++++++++++++++ 2 files changed, 182 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 411346d0..835d365a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -215,7 +215,22 @@ public static PersistedSymbolDict open(String slotDir) { */ public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; - long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; + boolean exists = ff.exists(filePath); + long existing = exists ? ff.length(filePath) : -1L; + if (exists && existing < 0) { + // The file is present but its length could not be stat'd (a transient EIO + // on a flaky disk). Do NOT fall through to openFresh below, which O_TRUNCs: + // truncating the only copy of load-bearing state on a TRANSIENT fault is the + // exact destruction the class-level "Never recreate over an existing file" + // note forbids -- and unlike the openExisting read path, this routing check + // otherwise has no guard. A genuine sub-header stub reports a length in + // [0, HEADER_SIZE); only a stat error reports < 0, so the two are + // distinguishable. Degrade to full self-sufficient frames and leave every + // byte on disk for a later attempt, once the transient clears. + LOG.warn("symbol dict {} exists but its length could not be read; " + + "falling back to full-dictionary frames (file left intact)", filePath); + return null; + } if (existing >= HEADER_SIZE) { // A dictionary at or past Integer.MAX_VALUE cannot be read back: // openExisting reads it into ONE int-sized native buffer, and the (int) @@ -310,36 +325,12 @@ public synchronized void appendRawEntries(long addr, int len, int count) { // with the entries it holds, shifting the dense id->symbol map on recovery. // The sole caller derives count and len from one beginMessage, so this cannot // fire today -- but the file this guards is the one the "resend required" - // contract rests on, so it stays. - long src = addr; - long srcLimit = addr + len; - for (int i = 0; i < count; i++) { - long symLen = 0; - int shift = 0; - while (src < srcLimit) { - byte b = Unsafe.getUnsafe().getByte(src++); - symLen |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - // A canonical entry-length varint is <= 5 bytes; a longer - // continuation run is corrupt. The bound check below rejects it. - break; - } - } - src += symLen; // src was just past the len varint - if (src > srcLimit) { - throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME - + " [entry=" + i + ", count=" + count + ']'); - } - } - if (src != srcLimit) { - throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " - + FILE_NAME + " [count=" + count + ", len=" + len - + ", consumed=" + (int) (src - addr) + ']'); - } + // contract rests on, so keep the structural guard. Gated behind assert: it + // re-walks every entry's length prefix on the common flush path, and the client + // library runs without -ea in production (embedded in user apps), so this holds + // the guard in the client's own -ea test suite without the per-flush cost in + // production. + assert validateRawEntries(addr, len, count); int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len); ensureScratch((long) hdrLen + len + CRC_SIZE); long p = NativeBufferWriter.writeVarint(scratchAddr, count); @@ -693,6 +684,48 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0); } + /** + * Verifies that {@code count} wire entries ({@code [len varint][utf8]}) occupy + * exactly {@code len} bytes from {@code addr}. Returns {@code true} when the triple + * is consistent; throws {@link IllegalStateException} (naming the offending entry / + * count / consumed bytes) otherwise. Called only from an {@code assert} in + * {@link #appendRawEntries}: it guards an internal invariant the sole caller cannot + * violate today, so it runs under the test suite's {@code -ea} but is elided from + * the per-flush production path (client apps run without {@code -ea}). + */ + private static boolean validateRawEntries(long addr, int len, int count) { + long src = addr; + long srcLimit = addr + len; + for (int i = 0; i < count; i++) { + long symLen = 0; + int shift = 0; + while (src < srcLimit) { + byte b = Unsafe.getUnsafe().getByte(src++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + // A canonical entry-length varint is <= 5 bytes; a longer + // continuation run is corrupt. The bound check below rejects it. + break; + } + } + src += symLen; // src was just past the len varint + if (src > srcLimit) { + throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME + + " [entry=" + i + ", count=" + count + ']'); + } + } + if (src != srcLimit) { + throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " + + FILE_NAME + " [count=" + count + ", len=" + len + + ", consumed=" + (int) (src - addr) + ']'); + } + return true; + } + /** * Grows the append scratch buffer to hold at least {@code required} bytes. *

    diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index c5222e0e..6c6dcf91 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -31,6 +31,7 @@ import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; import java.nio.charset.StandardCharsets; @@ -152,6 +153,56 @@ public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { }); } + @Test + public void testAppendRawEntriesRejectsInconsistentTripleUnderAssertions() throws Exception { + // appendRawEntries validates the (addr,len,count) triple before writing: an + // inconsistent triple would record a chunk whose stored entryCount disagreed + // with its entries and shift the dense id->symbol map on recovery, so it must + // fail loudly rather than corrupt the file. That check is an assert -- gated on + // -ea to keep the per-entry walk off the per-flush production path (client apps + // run without -ea) -- so observe it under the test suite's -ea, and skip if + // assertions happen to be disabled. + boolean assertionsEnabled = false; + assert assertionsEnabled = true; + Assume.assumeTrue("the guard is assert-gated; only observable under -ea", assertionsEnabled); + assertMemoryLeak(() -> { + Path src = Files.createTempDirectory("qwp-symdict-src"); + Path dst = Files.createTempDirectory("qwp-symdict-dst"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + dict.getOrAddSymbol("GOOG"); // id 2 + PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + encoded.appendSymbols(dict, 0, 2); + encoded.close(); + + // Grab the on-disk entry region (3 entries) verbatim, then feed it back + // with a count of 1: validateRawEntries stops one entry in with + // src < srcLimit and throws "under-filled", before any bytes are written. + PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + try { + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); + try { + raw.appendRawEntries(reopened.loadedEntriesAddr(), reopened.loadedEntriesLen(), 1); + Assert.fail("an inconsistent (len,count) triple must be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue("message names the under-filled buffer: " + expected.getMessage(), + expected.getMessage().contains("under-filled")); + Assert.assertEquals("a rejected triple must write nothing", 0, raw.size()); + } finally { + raw.close(); + } + } finally { + reopened.close(); + } + } finally { + rmDir(src); + rmDir(dst); + } + }); + } + @Test public void testAppendRawEntriesShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception { // The producer's FAST path persists a frame's already-encoded delta bytes @@ -774,6 +825,60 @@ public void testTornTrailingEntrySelfHeals() throws Exception { }); } + @Test + public void testTransientLengthFaultDegradesWithoutDestroyingTheFile() throws Exception { + // open() routes on ff.length(): a value < HEADER_SIZE falls through to the + // truncating openFresh(). A genuine sub-header stub reports a length in + // [0, HEADER_SIZE), but a TRANSIENT stat failure (an EIO on a flaky disk) + // reports the -1 error sentinel for a fully populated file -- and routing that + // to openFresh would O_TRUNC the only copy of load-bearing state, the exact + // destruction the "Never recreate over an existing file" contract forbids. A + // negative length is distinguishable from a stub, so open() must degrade to + // null (fall back to full self-sufficient frames) and leave every byte on disk + // for a later attempt, once the transient clears. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("one"); + d.appendSymbol("two"); + d.close(); + + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + Assert.assertTrue("a populated dict must exceed the header", before.length > HEADER_SIZE); + + // Reopen through a facade whose length() reports the -1 stat-error + // sentinel for the (present) file -- the branch under test. + PersistedSymbolDict reopened = PersistedSymbolDict.open(new StatFailsLengthFacade(), dir.toString()); + if (reopened != null) { + // Pre-fix: open() fell through to openFresh() and handed back a fresh + // empty dict over the now-truncated file. Close its fd so only the + // assertion below reports the failure, not a leaked descriptor. + reopened.close(); + } + Assert.assertNull("a populated dict whose length cannot be stat'd must degrade to null, not truncate", + reopened); + Assert.assertArrayEquals("a transient length-stat fault must NOT destroy the dictionary", + before, Files.readAllBytes(f)); + + // Once the filesystem recovers, the SAME file reopens its intact content. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + Assert.assertEquals("the intact content survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testTruncateFailureDegradesWithoutDestroyingTheFile() throws Exception { // A host crash can leave a torn/stale tail past the last complete chunk. @@ -894,4 +999,17 @@ public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } } + + /** + * Reports a length of -1 -- the stat-error sentinel -- for the dictionary file, + * reproducing a transient stat failure (an EIO on a flaky disk) where the file is + * present but its size cannot be read. open() must treat this as "present but + * unreadable" and degrade to null, NOT route it to the truncating fresh-open path. + */ + private static final class StatFailsLengthFacade extends DelegatingFilesFacade { + @Override + public long length(String path) { + return -1L; + } + } } From 0520dc47731d93c91527e9a41bb41f2658d9105a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 15:11:50 +0100 Subject: [PATCH 65/78] Don't quarantine a fully-acked torn-dict slot seedGlobalDictionaryFromPersisted walked collectReplaySymbolsAbove from FSN 0 and threw UnreplayableSlotException on the first symbol-dictionary gap regardless of ack state. For a recovered slot whose committed frames were all acked -- so nothing was left to replay -- a gap in that already delivered data still threw. (A host crash tore the unsynced side-file, or it could not be opened, while the low segments that registered the missing ids were trimmed away.) build() then set the slot aside as .unreplayable-, its fully-drained close deleted the delivered bytes the quarantine promises to preserve, and it logged a false "resend required" for data the server had already acknowledged. That gap only matters for frames that will REPLAY. When every committed frame has been acked (ackedFsn >= recoveredCommitBoundaryFsn), the seed now suppresses the throw: it seeds the intact prefix only, leaving the frame-derived symbols unused exactly as the send loop's mirror does on a -1, so the producer baseline and the mirror's sentDictCount still agree by construction, and resumes on the same slot. A genuine gap in UNACKED committed frames still throws, so the real-unreplayable quarantine path is unchanged. Add testFullyAckedTornSlotResumesInPlaceWithoutQuarantine, the acked counterpart to the existing unacked quarantine test: the same on-disk tear, but with the ack watermark stamped at the last committed frame, asserting no set-aside copy exists and the sender resumes in place. The test fails without the production change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 27 +++++++++ .../qwp/client/DeltaDictRecoveryTest.java | 56 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index afdb8114..8c954aba 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3933,6 +3933,33 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { ObjList fromFrames = new ObjList<>(); long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames); if (coverage < 0) { + // A gap: the surviving frames reference ids below their own delta start, + // introduced by frames since acked and trimmed away, and the persisted + // dictionary no longer holds them (a host crash tore its unsynced tail, or it + // could not be opened). That gap only matters for frames that will REPLAY. + // When every recovered committed frame is already acked + // (ackedFsn >= recoveredCommitBoundaryFsn), NOTHING replays: the gap is in + // data the server already has, and the retired orphan-deferred tail above the + // commit boundary is never transmitted. Throwing here would raise a false + // "resend required" for delivered data AND -- because such a slot is fully + // drained -- let build()'s connect-path close unlink the (already-delivered) + // bytes the quarantine claims to preserve. So DON'T throw: seed the intact + // prefix only, leaving fromFrames unused exactly as the send loop's mirror + // does on a -1 (it adds nothing), so the producer baseline and the mirror's + // sentDictCount still agree by construction. The producer resumes above the + // prefix and the fully-drained slot is cleaned up on close. + long ackedFsn = cursorEngine.ackedFsn(); + long commitBoundaryFsn = cursorEngine.recoveredCommitBoundaryFsn(); + if (ackedFsn >= commitBoundaryFsn) { + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + LOG.info("recovered store-and-forward slot has a torn/incomplete symbol dictionary, " + + "but every committed frame was already acked so nothing needs replaying; " + + "resuming on the intact prefix without quarantine and without data loss " + + "[recoveredPrefixSize={}, ackedFsn={}, commitBoundaryFsn={}]", + baseline, ackedFsn, commitBoundaryFsn); + return; + } + // Genuine loss: unacked committed frames reference ids nothing still holds. // Typed, because Sender.build() sets such a slot aside instead of failing: this // is the point at which every source of truth has been tried and none of them // holds the missing ids. See UnreplayableSlotException. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e8ff1e99..b4d91895 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -249,6 +249,62 @@ public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() th }); } + @Test + public void testFullyAckedTornSlotResumesInPlaceWithoutQuarantine() throws Exception { + // M1 regression -- the ACKED counterpart to + // testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside. The on-disk + // tear is IDENTICAL (earliest segment trimmed, .symbol-dict torn to its header, + // so the surviving frames' deltas start above ids nothing on disk still holds), + // but here every committed frame was already ACKED before the crash. Nothing is + // left to replay, so the "gap" is entirely in data the server already has. + // + // seedGlobalDictionaryFromPersisted must therefore NOT raise + // UnreplayableSlotException. Quarantining a fully-delivered slot would fire a + // false "resend required" alarm AND -- because such a slot is fully drained -- + // let build()'s connect-path close unlink the (already-delivered) bytes the + // quarantine claims to preserve. The slot must resume IN PLACE. + // + // Before the fix the gap detector ignored ack state and threw, so this slot was + // set aside as default.unreplayable-0 with a "resend required" error for data the + // server had already acknowledged. + assertMemoryLeak(() -> { + writeAndTearUnreplayableSlot(); + // Mark every committed frame acked. writeAndTearUnreplayableSlot writes 12 + // frames (FSNs 0..11), so stamping the watermark at 11 makes + // ackedFsn == recoveredCommitBoundaryFsn: a torn dictionary with nothing left + // to replay. (Not higher than 11 -- that would make the resuming producer's + // next frame look pre-acked.) + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), 11); + + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // build() must succeed WITHOUT setting the slot aside, and the producer + // must keep working on the SAME slot. + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow(); + s2.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { + Thread.sleep(20); + } + } + } + // The fully-acked slot was NOT quarantined: no set-aside copy exists (the + // inverse of assertUnreplayableSlotSetAside), and the sender resumed on the + // original slot. + Assert.assertFalse("a fully-acked torn slot must NOT be quarantined -- its data " + + "was already delivered, so there is nothing to resend", + java.nio.file.Files.isDirectory(Paths.get(sfDir, "default.unreplayable-0"))); + Assert.assertTrue("the sender must resume on the original slot", + java.nio.file.Files.isDirectory(slot)); + }); + } + private static int countSegmentFiles(java.nio.file.Path dir) { java.io.File[] files = dir.toFile().listFiles(); int n = 0; From 8d6a93243fadfaf57ed23d193e02f27b851fccec Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:04:47 +0100 Subject: [PATCH 66/78] Keep foreground cap-gap retries unbounded Separate symbol-dictionary catch-up cap-gap policy by loop owner. Foreground senders now retry until a compatible node returns, while orphan drainers retain bounded quarantine behavior. Preserve existing constructor signatures and add unit and integration coverage for retries beyond the orphan budget and recovery after the server cap is restored. --- .../main/java/io/questdb/client/Sender.java | 12 +- .../qwp/client/QwpWebSocketSender.java | 9 +- .../client/sf/cursor/BackgroundDrainer.java | 9 +- .../sf/cursor/CursorWebSocketSendLoop.java | 150 +++++++++++------- .../qwp/client/DeltaDictCatchUpTest.java | 74 +++------ ...WebSocketSendLoopCatchUpAlignmentTest.java | 74 +++++++-- 6 files changed, 197 insertions(+), 131 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 23fdf6a8..f73d3e35 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1760,7 +1760,7 @@ public Sender build() { /** * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must - * persist before the sender gives up on it and fails. + * persist before an orphan store-and-forward drainer quarantines its slot. *

    * A cap gap means a symbol already accepted by one node is too large to * re-register on the node the sender just failed over to, because that node @@ -1768,13 +1768,11 @@ public Sender build() { * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering * the cap below existing data. *

    - * The sender retries such a gap across reconnects rather than failing on sight, - * because the larger-cap node may simply be away -- a rolling restart is the most - * likely reason a failover happened at all. It gives up only once the gap has BOTH - * recurred many times AND persisted for this long, so an ordinary cluster - * operation cannot bring down a live producer. Raise it for a cluster whose + * A foreground sender retries such a gap indefinitely because the larger-cap node + * may simply be away. An orphan drainer may quarantine only once the gap has BOTH + * recurred many times AND persisted for this long. Raise it for a cluster whose * rolling restarts take longer than the 5-minute default; set it to {@code 0} to - * fail as soon as the retry count is exhausted. + * quarantine an orphan slot as soon as the retry count is exhausted. *

    * WebSocket transport only. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8c954aba..c22f01ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -314,9 +314,9 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; - // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before the - // send loop latches a terminal (connect-string key - // catchup_cap_gap_min_escalation_window_millis). See + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before an + // orphan drainer may quarantine its slot (connect-string key + // catchup_cap_gap_min_escalation_window_millis). Foreground senders retry forever. See // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. private long catchUpCapGapMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; @@ -3358,7 +3358,8 @@ private void ensureConnected() { durableAckKeepaliveIntervalMillis, maxFrameRejections, poisonMinEscalationWindowMillis, - catchUpCapGapMinEscalationWindowMillis); + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.CatchUpCapGapPolicy.RETRY_FOREVER); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index a88429ce..0e7bcd10 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -133,9 +133,9 @@ public final class BackgroundDrainer implements Runnable { // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this - // drainer's send loop latches a terminal. Same knob the foreground sender uses; the - // orphan drainer honours it too so a rolling restart cannot quarantine an otherwise - // drainable slot on a strike count alone. + // orphan drainer's send loop latches a terminal. Foreground senders retry forever; + // this bounded policy is permitted only so a persistent orphan slot can be + // quarantined for operator intervention. private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( @@ -584,7 +584,8 @@ public void run() { durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, poisonMinEscalationWindowMillis, - catchUpCapGapMinEscalationWindowMillis); + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET); loop.start(); while (!stopRequestedOrInterrupted()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index c705aaac..2d94dc76 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -136,7 +136,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; /** * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must - * persist before the loop latches a terminal, even once + * persist before an orphan drainer may latch a terminal, even once * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a * strike count measures "how many times did we look", not "how long has this been @@ -146,35 +146,34 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * It matters here because the cap gap's trigger IS a failover. With the reconnect * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling - * restart of the larger-cap node. A count-only budget would therefore hard-fail a - * live store-and-forward producer during a routine cluster operation, which is - * exactly the transient the budget exists to ride out. Requiring the dwell as well - * means the terminal needs BOTH "we have looked many times" AND "it has stayed true - * for a long time"; a node that returns inside the window ends the episode and the - * producer never notices. + * restart of the larger-cap node. A count-only budget could therefore quarantine an + * otherwise drainable orphan slot during a routine cluster operation. Requiring the + * dwell as well means the orphan terminal needs BOTH "we have looked many times" AND + * "it has stayed true for a long time". Foreground senders never apply this terminal + * policy: they retry indefinitely until a larger-cap node returns. *

    * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, * this codebase's existing notion of how long a transient outage may plausibly * last. Configurable per sender via the * {@code catchup_cap_gap_min_escalation_window_millis} connect-string key or * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} - * restores count-only escalation. + * restores count-only escalation for orphan drainers. */ public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts // -- catch-ups that reached a fresh server and found a single dictionary entry // too large for its advertised batch cap -- may occur, with no intervening - // SUCCESSFUL catch-up, before the sender latches a terminal. This is a SANCTIONED - // terminal (a genuine cluster batch-size capability gap), the connect-time analog - // of the orphan drainer's durable-ack capability gap + // SUCCESSFUL catch-up, before an orphan drainer latches a terminal. This is a + // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), + // the connect-time analog of the orphan drainer's durable-ack capability gap // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips // it -- an entry that fit its data frame under a cap always fits its bare catch-up // frame under that same cap -- so it only affects a heterogeneous / rolling-cap // cluster, where a failover to a smaller-cap node can hit it for an entry an - // earlier node accepted. Retrying rides out the transient window until a - // larger-cap node returns; only a persistent gap (this many cap gaps with no - // successful catch-up in between) latches. + // earlier node accepted. A foreground sender retries forever. An orphan drainer + // rides out the transient window until a larger-cap node returns; only a persistent + // gap (this many cap gaps with no successful catch-up in between) latches. // // Budget accounting (satisfies "a transient must never burn the terminal // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a @@ -222,11 +221,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); - // Pre-converted to nanos. Zero disables the dwell entirely (count-only escalation - // at MAX_CATCHUP_CAP_GAP_ATTEMPTS) -- the raw-constructor default; the user-facing - // 5-minute default is applied at the config layer, mirroring - // poisonMinEscalationWindowNanos. + // Pre-converted to nanos. Consulted only by the orphan terminal policy. Zero disables + // the dwell entirely (count-only escalation at MAX_CATCHUP_CAP_GAP_ATTEMPTS); the + // user-facing 5-minute default is applied at the config layer. private final long catchUpCapGapMinEscalationWindowNanos; + private final CatchUpCapGapPolicy catchUpCapGapPolicy; private final CursorSendEngine engine; private final long parkNanos; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only @@ -302,16 +301,15 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private int sentDictBytesCapacity; private int sentDictBytesLen; private int sentDictCount; - // Cap-gap attempts -- catch-ups that reached a node and found an entry too large - // for its batch cap -- since the last SUCCESSFUL catch-up (see - // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful - // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments - // nor resets it. NOT reset per connection -- it measures the cap-gap episode - // across reconnects so a persistent gap eventually latches. I/O-thread-only. + // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry + // too large for its batch cap -- since the last SUCCESSFUL catch-up (see + // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). Foreground retries + // never increment it. A successful catch-up resets it; a transient reconnect neither + // increments nor resets it. I/O-thread-only. private int catchUpCapGapAttempts; - // System.nanoTime() of the FIRST cap gap of the current episode. Anchors the - // escalation dwell. Reset together with catchUpCapGapAttempts on a successful - // catch-up, so a node that accepts the dictionary ends the episode outright. + // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. + // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a + // successful catch-up, so a node that accepts the dictionary ends the episode. // Anchoring at the FIRST gap (not at loop entry, and not refreshed per attempt) is // what keeps a TRANSIENT from burning the terminal budget: a transient never reaches // the catch-up, so it neither increments the counter nor moves the anchor -- it can @@ -592,13 +590,9 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, /** * Twelve-arg overload — omits the symbol-dict cap-gap escalation dwell, which - * defaults to {@code 0} (legacy: escalate as soon as - * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue, with no minimum - * wall-clock). Like the poison dwell, the user-facing default - * ({@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS}) is applied at the - * config layer (Sender / QwpWebSocketSender), so this raw constructor stays - * unopinionated for tests and internal callers that want deterministic strike-count - * escalation. + * defaults to {@code 0}. This and the thirteen-arg overload use the foreground-safe + * retry-forever policy; orphan drainers opt into count-and-dwell escalation through + * the policy-aware master constructor. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -618,7 +612,11 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } /** - * Master constructor — also accepts the poison-frame detector threshold + * Thirteen-arg overload. Catch-up cap gaps are retried indefinitely, which is the + * required policy for a foreground store-and-forward sender. Orphan drainers use the + * policy-aware overload below to opt into bounded terminal escalation. + *

    + * Also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum @@ -626,11 +624,10 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = legacy * immediate escalation at the threshold). *

    - * The final argument is the analogous dwell for the symbol-dict catch-up cap gap - * ({@code catchup_cap_gap_min_escalation_window_millis}; {@code >= 0}, where 0 = - * escalate as soon as {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue). See - * {@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS} for why a strike - * count alone is not a safe escalation signal on a live producer. + * The final argument is the analogous dwell for the symbol-dict catch-up cap gap. + * It is retained here for source and binary compatibility but is consulted only when + * the policy-aware overload selects + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET}. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -643,6 +640,32 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, int maxHeadFrameRejections, long poisonMinEscalationWindowMillis, long catchUpCapGapMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy.RETRY_FOREVER); + } + + /** + * Policy-aware master constructor. Foreground senders must pass + * {@link CatchUpCapGapPolicy#RETRY_FOREVER}; only orphan drainers may pass + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET} and quarantine a slot + * after the attempt and dwell thresholds are both exhausted. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy catchUpCapGapPolicy) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -658,6 +681,10 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.catchUpCapGapMinEscalationWindowNanos = catchUpCapGapMinEscalationWindowMillis * 1_000_000L; + if (catchUpCapGapPolicy == null) { + throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); + } + this.catchUpCapGapPolicy = catchUpCapGapPolicy; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -2440,8 +2467,8 @@ private int sendDictCatchUp() { long entryBytes = entryEnd - entryStart; // The exact table-less frame sendCatchUpChunk would build for THIS entry // alone: header + deltaStart varint (the entry's own global id) + - // deltaCount varint (1) + the entry bytes. Terminal only when even that - // solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // deltaCount varint (1) + the entry bytes. A cap gap exists only when even + // that solo frame exceeds the cap -- i.e. the entry genuinely cannot be // re-registered. Testing the real solo frame (not the conservative // packing budget above) is what keeps a HOMOGENEOUS cluster // livelock-free: an entry the producer already shipped in a data frame @@ -2458,18 +2485,17 @@ private int sendDictCatchUp() { // catch-up frame under that same cap), so the only way in is a // heterogeneous / rolling-cap failover to a smaller-cap node. // - // Give the cluster a settle budget instead of latching on first - // sight: a larger-cap node may return, so retry across reconnects - // and only latch a terminal after MAX_CATCHUP_CAP_GAP_ATTEMPTS - // consecutive attempts still find it too large. Under budget the + // A foreground sender retries indefinitely: store-and-forward must + // contain a transient heterogeneous-cluster window instead of surfacing + // it to the producer. Only an orphan drainer may apply the settle budget + // below and quarantine its slot after a persistent gap. Under budget the // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with - // backoff and re-runs the catch-up, which resets the counter on a - // node that accepts it. A transient reconnect (connect/upgrade - // failure, role reject) never reaches the catch-up, so it neither - // increments nor burns this budget. On exhaustion latch via - // recordFatal, NOT fail() -- failing from inside the catch-up would - // re-enter connectLoop (see CatchUpSendException); the data must be - // resent after the cap is raised. + // backoff and re-runs the catch-up, which resets the counter on a node + // that accepts it. A transient reconnect (connect/upgrade failure, role + // reject) never reaches the catch-up, so it neither increments nor burns + // the orphan budget. On exhaustion latch via recordFatal, NOT fail() -- + // failing from inside the catch-up would re-enter connectLoop (see + // CatchUpSendException); the data must be resent after the cap is raised. // Escalation needs BOTH the strike count AND a minimum wall-clock dwell // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count // alone measures how often we looked, not how long the gap has held -- @@ -2487,6 +2513,13 @@ private int sendDictCatchUp() { // the terminal never latches, however long the gap truly persists. // recordHeadRejectionStrike keys its episode off poisonStrikes for the same // reason. + if (catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { + throw new CatchUpSendException(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' + + "; retrying indefinitely -- a larger-cap node may return")); + } + long nowNanos = System.nanoTime(); if (catchUpCapGapAttempts == 0) { catchUpCapGapFirstNanos = nowNanos; @@ -2815,6 +2848,17 @@ private boolean tryRetireOrphanTail() { return true; } + /** + * Determines whether an oversized symbol-dictionary catch-up entry is always + * retriable or may become terminal after the orphan settle budget. + */ + public enum CatchUpCapGapPolicy { + /** Keep reconnecting until a node with a compatible batch cap returns. */ + RETRY_FOREVER, + /** Quarantine an orphan slot after both the attempt and dwell limits expire. */ + TERMINAL_AFTER_SETTLE_BUDGET + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index db7603f2..a7b2fb5f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -25,7 +25,6 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -210,17 +209,11 @@ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exce } @Test - public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { + public void testForegroundCatchUpCapGapRetriesPastOrphanBudgetAndRecovers() throws Exception { // A dictionary entry that exceeds the reconnect server's per-chunk budget - // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. Every - // reachable node re-advertises the same small cap here, so the gap never - // resolves: sendDictCatchUp must retry across the settle budget and then - // latch a clean terminal ("... during catch-up") rather than call fail() - // (which from inside the catch-up re-enters connectLoop). Pre-fix it latched - // on the FIRST cap gap; the settle budget (MAX_CATCHUP_CAP_GAP_ATTEMPTS) - // rides out a transient smaller-cap window first (see the retry sibling in - // CursorWebSocketSendLoopCatchUpAlignmentTest), and only a persistent gap - // exhausts it. Small reconnect backoffs keep the budgeted attempts fast. + // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. A live + // foreground sender must keep retrying without surfacing that cluster state to + // the producer, even after the orphan drainer's 16-attempt quarantine budget. // // Connection 1 advertises no cap, so the ~202-byte symbol registers and // enters the sent-dictionary mirror. The handler then shrinks the @@ -237,49 +230,27 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry - LineSenderException terminal = null; - // catchup_cap_gap_min_escalation_window_millis=0 restores count-only - // escalation, so the terminal latches as soon as the settle budget's - // strikes are exhausted. The DEFAULT is a 5-minute wall-clock dwell on - // top of the strike count, precisely so a cap gap that lasts only as - // long as a rolling restart cannot hard-fail a live producer -- and a - // test cannot wait that out. Zeroing it here is what makes the terminal - // observable; the dwell itself is covered by - // CursorWebSocketSendLoopCatchUpAlignmentTest. - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + // Zeroing the dwell makes the old faulty foreground policy terminal at + // its 16th cap gap. Reaching 20 handshakes therefore proves the live loop + // is no longer governed by the orphan budget. + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50" - + ";catchup_cap_gap_min_escalation_window_millis=0;"); - try { + + ";catchup_cap_gap_min_escalation_window_millis=0;")) { sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); sender.flush(); - // The terminal latches on the I/O thread once the reconnect's - // catch-up hits the oversized entry; it surfaces to the producer - // on a subsequent flush. Poll a bounded time for it. The polling - // rows use a small symbol that fits the shrunk cap, so the - // producer-side cap check never fires and flush() surfaces the - // I/O thread's catch-up terminal via checkError. - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); - sender.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - sender.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } - } + waitFor(() -> server.handshakeCount() >= 20, 10_000); + + // Producer calls remain usable while the wire is stuck in the cap + // gap. Then restore the larger-cap node and prove the buffered row is + // actually acknowledged, not merely accepted into the local SF ring. + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + server.setAdvertisedMaxBatchSize(0); + sender.table("t").symbol("s", "z").longColumn("v", 3L).atNow(); + long targetFsn = sender.flushAndGetSequence(); + Assert.assertTrue("foreground sender must recover and drain after the cap is restored", + sender.awaitAckedFsn(targetFsn, 5_000)); } - Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); - Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), - terminal.getMessage().contains("during catch-up")); } }); } @@ -499,7 +470,7 @@ private static int tableCount(byte[] frame) { * registers), then -- once connection 1 has sent something -- shrinks the * advertised batch cap and drops the socket. The reconnect (connection 2) * therefore advertises a cap whose catch-up budget is too small for the - * symbol, exercising the oversized-entry terminal in sendDictCatchUp. + * symbol, exercising the foreground retry path in sendDictCatchUp. */ private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); @@ -516,6 +487,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (currentClient != client) { currentClient = client; connectionsAccepted.incrementAndGet(); + nextSeq.set(0); } try { client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 4e00e1e6..fed94645 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -267,17 +267,51 @@ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { }); } + @Test + public void testForegroundCatchUpCapGapRetriesPastOrphanBudget() throws Exception { + // The foreground policy must never accrue or exhaust the orphan drainer's + // quarantine budget. Drive more cap gaps than that entire budget and assert every + // failure remains retriable to the I/O loop and invisible to the producer. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newForegroundLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + for (int i = 1; i <= maxAttempts + 4; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); + } + assertEquals("foreground retries must not burn the orphan attempt budget", + 0, readInt(loop, "catchUpCapGapAttempts")); + assertEquals("foreground retries must not anchor an orphan cap-gap episode", + -1L, readLong(loop, "catchUpCapGapFirstNanos")); + } finally { + loop.close(); + } + } + }); + } + @Test public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() throws Exception { // The strike count alone must NOT latch a terminal: escalation also requires the // cap-gap episode to have persisted for catchUpCapGapMinEscalationWindowMillis. // - // This is what keeps a routine rolling restart from killing a live producer. + // This keeps a routine rolling restart from quarantining a drainable orphan slot. // MAX_CATCHUP_CAP_GAP_ATTEMPTS strikes accrue in ~2 minutes at the capped // reconnect backoff -- less than the time the larger-cap node is away -- so a - // count-only budget would hard-fail the sender on the very transient the budget + // count-only budget would quarantine the slot on the very transient the budget // exists to ride out. Here we drive far MORE than the budget's strikes inside a - // (deliberately huge) window and assert the sender stays alive. + // deliberately huge window and assert the orphan loop stays retriable. TestUtils.assertMemoryLeak(() -> { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); maxField.setAccessible(true); @@ -446,7 +480,7 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // the reconnect loop rides it out -- a larger-cap node may return -- and only // after MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive cap gaps does it recordFatal. // Pre-fix the first cap gap latched a terminal, so one transient failover to a - // smaller-cap node killed the sender. (A successful catch-up resets the budget; + // smaller-cap node quarantined the orphan slot. (A successful catch-up resets the budget; // the other catch-up tests, which use a fitting cap, never trip it.) TestUtils.assertMemoryLeak(() -> { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); @@ -456,13 +490,13 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // retriable loop below is bounded by maxAttempts, so keying this test purely // off the constant under test makes it TAUTOLOGICAL: a regression of // MAX_CATCHUP_CAP_GAP_ATTEMPTS to 1 -- which is precisely the pre-fix bug this - // test names, a single cap gap killing the sender -- would run the loop ZERO + // test names, a single cap gap quarantining the slot -- would run the loop ZERO // times, the "exhausting" attempt would become the FIRST attempt, and the test // would still pass green. Requiring > 1 makes that regression fail here, and it // also guarantees the loop runs at least once, so the first cap gap is genuinely // asserted retriable rather than vacuously skipped. assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap, else a single " - + "transient failover to a smaller-cap node kills the sender " + + "transient failover to a smaller-cap node quarantines the slot " + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']', maxAttempts > 1); // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol. @@ -512,7 +546,7 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { // gaps across reconnects; a successful catch-up ends the episode and MUST reset // it to 0 (sendDictCatchUp's final line). Otherwise cap gaps interspersed with // successful catch-ups -- a rolling-cap cluster where a larger-cap node comes - // and goes -- would accumulate to a spurious terminal over a long-lived sender. + // and goes -- would accumulate to a spurious terminal over a long-lived orphan drainer. // testCatchUpCapGapRetriesUntilBudgetThenLatches only accrues gaps under one // fixed cap with no success interleaved, so it cannot pin the reset. TestUtils.assertMemoryLeak(() -> { @@ -552,7 +586,7 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { // Behavioural proof the budget is genuinely fresh: max-1 more cap // gaps still latch NO terminal (they would if the counter had stayed - // at max-1 -- one more gap would have hit the cap and killed the sender). + // at max-1 -- one more gap would have quarantined the slot). client.cap = 160; for (int i = 1; i < maxAttempts; i++) { try { @@ -716,9 +750,8 @@ private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient /** * As {@link #newLoop(CursorSendEngine, WebSocketClient)} but with an explicit - * cap-gap escalation dwell. {@code 0} (the default here) means count-only - * escalation, which is what the raw constructor gives and what the strike-count - * tests want; the user-facing 5-minute default is applied at the config layer. + * cap-gap escalation dwell. These white-box tests model an orphan drainer, where + * {@code 0} means count-only quarantine; foreground loops retry indefinitely. */ private CursorWebSocketSendLoop newLoop( CursorSendEngine engine, WebSocketClient client, long capGapWindowMillis @@ -731,7 +764,24 @@ private CursorWebSocketSendLoop newLoop( 5_000L, 100L, 5_000L, false, CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - 0L, capGapWindowMillis); + 0L, capGapWindowMillis, + CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET); + } + + private CursorWebSocketSendLoop newForegroundLoop( + CursorSendEngine engine, WebSocketClient client + ) { + // Deliberately use the compatibility overload: its safe default must remain the + // foreground RETRY_FOREVER policy for external callers. + return new CursorWebSocketSendLoop( + client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + throw new UnsupportedOperationException("test loop is never started"); + }, + 5_000L, 100L, 5_000L, false, + CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, 0L); } private CursorSendEngine newEngine() { From 84293fa1c92b61398c7f7b0ec0dc905acbaad130 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:20:35 +0100 Subject: [PATCH 67/78] Reset cap-gap budget across unrelated outages --- .../sf/cursor/CursorWebSocketSendLoop.java | 88 +++++++++----- ...WebSocketSendLoopCatchUpAlignmentTest.java | 113 +++++++++++++++++- 2 files changed, 170 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2d94dc76..441edd90 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -163,8 +163,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts // -- catch-ups that reached a fresh server and found a single dictionary entry - // too large for its advertised batch cap -- may occur, with no intervening - // SUCCESSFUL catch-up, before an orphan drainer latches a terminal. This is a + // too large for its advertised batch cap -- may occur consecutively before an + // orphan drainer latches a terminal. This is a // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), // the connect-time analog of the orphan drainer's durable-ack capability gap // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips @@ -173,18 +173,17 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // cluster, where a failover to a smaller-cap node can hit it for an entry an // earlier node accepted. A foreground sender retries forever. An orphan drainer // rides out the transient window until a larger-cap node returns; only a persistent - // gap (this many cap gaps with no successful catch-up in between) latches. + // gap (this many consecutive cap gaps with no successful catch-up or unrelated + // reconnect state in between) latches. // // Budget accounting (satisfies "a transient must never burn the terminal // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a - // node is reached and an entry is oversized, and resets ONLY when a catch-up fully - // succeeds. A TRANSIENT reconnect (connect refuse, upgrade/role failure) never - // reaches the catch-up, so it NEITHER increments nor resets the counter -- it only - // lengthens the wall-clock settle window. The terminal therefore always requires - // this many GENUINE cap gaps; a transient can never inflate it. Deliberately NOT - // reset on a mere successful RECONNECT: a reconnect to the small-cap node itself - // produces the cap gap, so resetting there would stop a persistent gap from ever - // latching -- the reset must gate on a successful CATCH-UP, not on connecting. + // node is reached and an entry is oversized. A successful catch-up ends the + // episode, as does any unrelated reconnect state (connect refusal, catch-up send + // failure, upgrade/role rejection): otherwise its downtime would count toward the + // wall-clock dwell even though no cap gap was observed. The cap-gap exception + // itself does NOT reset the episode, so consecutive small-cap nodes still prove a + // persistent cluster capability gap and can exhaust the orphan policy. private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even @@ -302,19 +301,17 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private int sentDictBytesLen; private int sentDictCount; // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry - // too large for its batch cap -- since the last SUCCESSFUL catch-up (see - // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). Foreground retries - // never increment it. A successful catch-up resets it; a transient reconnect neither - // increments nor resets it. I/O-thread-only. + // too large for its batch cap -- with no intervening successful catch-up or unrelated + // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). + // Foreground retries never increment it. I/O-thread-only. private int catchUpCapGapAttempts; // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a - // successful catch-up, so a node that accepts the dictionary ends the episode. - // Anchoring at the FIRST gap (not at loop entry, and not refreshed per attempt) is - // what keeps a TRANSIENT from burning the terminal budget: a transient never reaches - // the catch-up, so it neither increments the counter nor moves the anchor -- it can - // only lengthen the wall clock, which alone can never latch the terminal because the - // strike count still has to be satisfied. I/O-thread-only, like catchUpCapGapAttempts. + // successful catch-up or an unrelated reconnect state, so only an uninterrupted run + // of cap-gap observations contributes wall-clock dwell. Anchoring at the FIRST gap + // (not refreshed per attempt) lets consecutive small-cap nodes satisfy the dwell; + // resetting after transport/role states prevents their downtime from doing so. + // I/O-thread-only, like catchUpCapGapAttempts. // // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant @@ -1436,6 +1433,15 @@ private void clearDurableAckTracking() { lastFrameOrPingNanos = 0L; } + private static boolean isCatchUpCapGap(Throwable t) { + return t instanceof CatchUpSendException && ((CatchUpSendException) t).capGap; + } + + private void resetCatchUpCapGapEpisode() { + catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; + } + /** * Shared per-outage retry loop. Used by {@link #fail(Throwable)} for * mid-flight wire failures (phase="reconnect") and by @@ -1450,6 +1456,13 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + // A wire/role state that interrupted an orphan cap-gap run is not evidence that + // the cluster's batch cap stayed incompatible during the outage. Start a fresh + // episode; the cap-gap exception itself is the one reconnect cause that preserves + // the existing attempt count and dwell anchor. + if (!isCatchUpCapGap(initial)) { + resetCatchUpCapGapEpisode(); + } LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); @@ -1519,6 +1532,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, fsnAtZero); return; } + // A null factory result is an unsuccessful connect state, not a cap-gap + // observation. Do not let the time spent retrying it satisfy orphan dwell. + resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // Terminal across all configured endpoints per spec sf-client.md // section 13.3: auth (401/403) bypasses reconnect and surfaces as @@ -1595,6 +1611,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // endpoint, forever. Growing to reconnectMaxBackoffMillis // mirrors the orphan drainer's role-reject path and honours the // documented capped-exponential-backoff contract. + resetCatchUpCapGapEpisode(); lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1618,6 +1635,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(e); throw (Error) e; } + if (!isCatchUpCapGap(e)) { + resetCatchUpCapGapEpisode(); + } lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -2490,10 +2510,10 @@ private int sendDictCatchUp() { // it to the producer. Only an orphan drainer may apply the settle budget // below and quarantine its slot after a persistent gap. Under budget the // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with - // backoff and re-runs the catch-up, which resets the counter on a node - // that accepts it. A transient reconnect (connect/upgrade failure, role - // reject) never reaches the catch-up, so it neither increments nor burns - // the orphan budget. On exhaustion latch via recordFatal, NOT fail() -- + // backoff and re-runs the catch-up, which resets the episode on a node + // that accepts it. An unrelated transport/upgrade/role state also resets + // the episode, so its downtime cannot satisfy the orphan dwell. On + // exhaustion latch via recordFatal, NOT fail() -- // failing from inside the catch-up would re-enter connectLoop (see // CatchUpSendException); the data must be resent after the cap is raised. // Escalation needs BOTH the strike count AND a minimum wall-clock dwell @@ -2517,7 +2537,7 @@ private int sendDictCatchUp() { throw new CatchUpSendException(new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' - + "; retrying indefinitely -- a larger-cap node may return")); + + "; retrying indefinitely -- a larger-cap node may return"), true); } long nowNanos = System.nanoTime(); @@ -2540,7 +2560,7 @@ private int sendDictCatchUp() { if (exhausted) { recordFatal(err); } - throw new CatchUpSendException(err); + throw new CatchUpSendException(err, true); } if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); @@ -2563,8 +2583,7 @@ private int sendDictCatchUp() { // count and the wall-clock anchor. Resetting only the count would leave a stale // anchor from an old episode, so the very first strike of a LATER episode would // already satisfy the dwell. - catchUpCapGapAttempts = 0; - catchUpCapGapFirstNanos = -1L; + resetCatchUpCapGapEpisode(); return framesSent; } @@ -2888,11 +2907,20 @@ public interface ReconnectFactory { * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} * from the I/O loop body (trySendOne path), or dropping the dead client so * the I/O thread reconnects (start path). A JVM {@code Error} is never - * wrapped -- it must stay terminal. + * wrapped -- it must stay terminal. The {@code capGap} marker lets the reconnect + * loop preserve only consecutive incompatible-cap observations; ordinary catch-up + * send failures restart the orphan settle episode. */ private static final class CatchUpSendException extends RuntimeException { + private final boolean capGap; + CatchUpSendException(Throwable cause) { + this(cause, false); + } + + CatchUpSendException(Throwable cause, boolean capGap) { super(cause); + this.capGap = capGap; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index fed94645..43bd9972 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -27,6 +27,7 @@ import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; @@ -40,12 +41,12 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; -import java.util.concurrent.TimeUnit; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -472,6 +473,16 @@ public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Excep }); } + @Test + public void testTransportOutageRestartsCapGapEpisode() throws Exception { + assertUnrelatedReconnectStateRestartsCapGapEpisode(false); + } + + @Test + public void testRoleRejectRestartsCapGapEpisode() throws Exception { + assertUnrelatedReconnectStateRestartsCapGapEpisode(true); + } + @Test public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // M1: an entry too large for the fresh server's cap during catch-up (a @@ -660,6 +671,76 @@ private static void appendFrames(CursorSendEngine engine, int count) { } } + private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReject) throws Exception { + // Accrue an orphan drainer's cap-gap strikes to one short of terminal, then + // simulate a long unrelated outage before another small-cap node appears. The + // outage must end the old episode: its wall-clock duration says nothing about + // whether the cluster's batch cap remained incompatible while no node answered. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + assertTrue("the cap-gap settle budget must have a retriable interval", maxAttempts > 1); + + int[] reconnectCalls = {0}; + long[] staleAnchor = {Long.MIN_VALUE}; + CursorWebSocketSendLoop[] loopRef = new CursorWebSocketSendLoop[1]; + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + int call = ++reconnectCalls[0]; + if (call < maxAttempts) { + return new CatchUpCapturingClient(160); + } + if (call == maxAttempts) { + assertEquals("precondition: consecutive cap gaps survive reconnect", + maxAttempts - 1, + readInt(loopRef[0], "catchUpCapGapAttempts")); + // Model the elapsed outage without sleeping. With the defect, + // this old anchor survives the unrelated failure and the next + // cap gap immediately satisfies both terminal conditions. + staleAnchor[0] = System.nanoTime() - TimeUnit.HOURS.toNanos(2); + setField(loopRef[0], "catchUpCapGapFirstNanos", staleAnchor[0]); + if (roleReject) { + throw new QwpRoleMismatchException( + "PRIMARY", null, "all endpoints role-rejected"); + } + throw new LineSenderException("transport unavailable"); + } + if (call == maxAttempts + 1) { + // Stop after getServerMaxBatchSize() has driven the final cap + // gap, leaving its fresh episode state observable below. + return new CatchUpCapturingClient(160, false, + () -> setBooleanFieldUnchecked(loopRef[0], "running", false)); + } + throw new AssertionError("unexpected reconnect call " + call); + }, + 5_000L, 0L, 0L, false, + CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, TimeUnit.HOURS.toMillis(1), + CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET); + loopRef[0] = loop; + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + setBooleanField(loop, "running", true); + invokeConnectLoop(loop); + + loop.checkError(); + assertEquals("pre-outage cap gaps must not carry into the new episode", + 1, readInt(loop, "catchUpCapGapAttempts")); + assertTrue("the post-outage cap gap must get a fresh dwell anchor", + readLong(loop, "catchUpCapGapFirstNanos") > staleAnchor[0]); + assertEquals("test must observe gaps, the unrelated state, and a new gap", + maxAttempts + 1, reconnectCalls[0]); + } finally { + loop.close(); + } + } + }); + } + // Builds a QWP delta frame [12-byte header][deltaStart varint][deltaCount // varint][ [len varint][utf8] ... ] for the given symbols. accumulateSentDict // skips the header, so its content is irrelevant; the caller frees the frame. @@ -744,6 +825,13 @@ private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loo m.invoke(loop, replayStart); } + private static void invokeConnectLoop(CursorWebSocketSendLoop loop) throws Exception { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "connectLoop", Throwable.class, String.class, long.class); + m.setAccessible(true); + m.invoke(loop, new LineSenderException("test reconnect"), "reconnect", 0L); + } + private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { return newLoop(engine, client, 0L); } @@ -830,6 +918,20 @@ private static void setIntField(Object target, String name, int value) throws Ex f.setInt(target, value); } + private static void setBooleanField(Object target, String name, boolean value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setBoolean(target, value); + } + + private static void setBooleanFieldUnchecked(Object target, String name, boolean value) { + try { + setBooleanField(target, name, value); + } catch (Exception e) { + throw new AssertionError(e); + } + } + private static int varintSize(long value) { int n = 1; while (value > 0x7F) { @@ -856,6 +958,7 @@ private static final class CatchUpCapturingClient extends WebSocketClient { // Mutable so a test can model a rolling-cap cluster: raise it for a node that // accepts the dictionary, lower it for a smaller-cap node that cap-gaps. private int cap; + private final Runnable onCapRead; private final boolean throwOnSend; private int framesSent; @@ -864,13 +967,21 @@ private static final class CatchUpCapturingClient extends WebSocketClient { } CatchUpCapturingClient(int cap, boolean throwOnSend) { + this(cap, throwOnSend, null); + } + + CatchUpCapturingClient(int cap, boolean throwOnSend, Runnable onCapRead) { super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); this.cap = cap; this.throwOnSend = throwOnSend; + this.onCapRead = onCapRead; } @Override public int getServerMaxBatchSize() { + if (onCapRead != null) { + onCapRead.run(); + } return cap; } From fe9546fad32a03d487cd91f03e23802b55563ed7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:39:41 +0100 Subject: [PATCH 68/78] Prevent orphan adoption during slot quarantine --- .../main/java/io/questdb/client/Sender.java | 184 ++++++++++-------- .../client/sf/cursor/BackgroundDrainer.java | 53 ++++- .../qwp/client/sf/cursor/SlotLock.java | 74 +++++-- .../qwp/client/DeltaDictRecoveryTest.java | 106 +++++++++- .../client/sf/cursor/OrphanScannerTest.java | 25 +++ .../qwp/client/sf/cursor/SlotLockTest.java | 24 +++ 6 files changed, 362 insertions(+), 104 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index f73d3e35..be4fe409 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -40,6 +40,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; @@ -1012,6 +1013,8 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + @TestOnly + private static volatile Runnable quarantineAfterCloseHook; // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the // sender's own slot name, so a restarted sender does not re-adopt it as its own; // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and @@ -1559,91 +1562,101 @@ public Sender build() { sfAppendDeadlineMillis == PARAMETER_NOT_SET_EXPLICITLY ? CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS : sfAppendDeadlineMillis * 1_000_000L; - CursorSendEngine cursorEngine = new CursorSendEngine( - slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? errorInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; - int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? connectionListenerInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; - List wsEndpoints = - new ArrayList<>(hosts.size()); - for (int i = 0, n = hosts.size(); i < n; i++) { - wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); - } - // The recovery seed inside connect() is the authority on whether a recovered - // slot can be replayed: it rebuilds the dictionary from its intact prefix and - // then from the surviving frames' own delta sections, and throws - // UnreplayableSlotException only once neither source holds the missing ids. - // Quarantining on anything weaker would set aside slots that recovery can - // still rescue, so build() waits for that verdict rather than pre-judging it. QwpWebSocketSender connected = null; - boolean quarantined = false; - while (connected == null) { - try { - connected = QwpWebSocketSender.connect( - wsEndpoints, - wsTlsConfig, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - wsAuthHeader, - requestDurableAck, - cursorEngine, - actualCloseFlushTimeoutMillis, - actualReconnectMaxDurationMillis, - actualReconnectInitialBackoffMillis, - actualReconnectMaxBackoffMillis, - actualInitialConnectMode, - errorHandler, - actualErrorInboxCapacity, - actualDurableAckKeepaliveIntervalMillis, - authTimeoutMillis, - connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, - connectionListener, - actualConnectionListenerInboxCapacity, - actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis, - actualCatchUpCapGapMinEscalationWindowMillis - ); - } catch (UnreplayableSlotException e) { - // The one failure build() recovers from. The slot's frames reference ids - // that nothing still holds, so they can never go on the wire -- but that is - // no reason to take the producer down with them. Before this, the throw - // escaped build() and, because senderId is stable and a not-fully-drained - // slot is retained on close, every retry re-recovered the same slot and - // threw again: the application could not construct a Sender at all, so it - // could not even BUFFER new rows. An already-lost batch became an unbounded - // outage of everything after it. - // - // Set the slot aside instead, keep its bytes for forensics and resend, and - // start the producer on a clean one. Once only: a second such failure would - // mean the FRESH slot is unreplayable, which cannot happen, so let it out - // rather than loop. - if (quarantined || slotPath == null) { + // The parent-anchored logical lock is stable across a slot rename. Keep it + // from before the directory-local lock is acquired until connect() has either + // adopted that engine or quarantine has closed, renamed and recreated it. + // This closes the inode-swap window in which an already-queued orphan drainer + // could otherwise acquire the renamed directory's old .lock and later operate + // on the fresh slot through the original pathname. + try (SlotLock logicalSlotLock = slotPath == null + ? null + : SlotLock.acquireLogical(slotPath)) { + CursorSendEngine cursorEngine = new CursorSendEngine( + slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? errorInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; + int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? connectionListenerInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; + List wsEndpoints = + new ArrayList<>(hosts.size()); + for (int i = 0, n = hosts.size(); i < n; i++) { + wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); + } + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + boolean quarantined = false; + while (connected == null) { try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort + connected = QwpWebSocketSender.connect( + wsEndpoints, + wsTlsConfig, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + wsAuthHeader, + requestDurableAck, + cursorEngine, + actualCloseFlushTimeoutMillis, + actualReconnectMaxDurationMillis, + actualReconnectInitialBackoffMillis, + actualReconnectMaxBackoffMillis, + actualInitialConnectMode, + errorHandler, + actualErrorInboxCapacity, + actualDurableAckKeepaliveIntervalMillis, + authTimeoutMillis, + connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, + connectionListener, + actualConnectionListenerInboxCapacity, + actualMaxFrameRejections, + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis + ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + } catch (Throwable t) { + // connect() failed before ownership of cursorEngine + // transferred — close it ourselves. + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw t; } - throw e; } - quarantined = true; - cursorEngine = quarantineTornSlot( - cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - } catch (Throwable t) { - // connect() failed before ownership of cursorEngine - // transferred — close it ourselves. - try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort - } - throw t; - } } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY @@ -3047,6 +3060,10 @@ private static CursorSendEngine quarantineTornSlot( // path already closed the engine; close() is idempotent, so make it explicit rather // than depend on that. torn.close(); + Runnable hook = quarantineAfterCloseHook; + if (hook != null) { + hook.run(); + } String quarantinePath = null; for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { @@ -3074,6 +3091,11 @@ private static CursorSendEngine quarantineTornSlot( return new CursorSendEngine(slotPath, sfMaxBytes, sfMaxTotalBytes, sfAppendDeadlineNanos); } + @TestOnly + public static void setQuarantineAfterCloseHookForTest(Runnable hook) { + quarantineAfterCloseHook = hook; + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 0e7bcd10..8e287b4a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -44,7 +44,9 @@ *

    * Lifecycle: *

      - *
    1. Acquire the slot's {@code .lock}; skip silently on contention.
      2. + *
    2. Acquire the parent-anchored logical-slot lock and revalidate the + * scanner snapshot; skip silently on contention or a stale snapshot.
      3. + *
    3. Acquire the slot's {@code .lock}, then release the logical lock.
      4. *
    4. Open a {@link CursorSendEngine} on the slot — recovery picks up * every {@code .sfa} file already on disk.
      5. *
    5. Open a fresh {@link WebSocketClient} via the supplied factory @@ -529,27 +531,54 @@ private boolean stopRequestedOrInterrupted() { @Override public void run() { runnerThread = Thread.currentThread(); + SlotLock logicalSlotLock = null; CursorSendEngine engine = null; WebSocketClient client = null; CursorWebSocketSendLoop loop = null; try { - // The engine acquires the slot's .lock itself — we don't need - // (and must not) double-lock it. If another sender or drainer - // holds it, the engine constructor throws and we exit silently - // (no .failed sentinel — contention is expected, not an error). + // Scanner results are only snapshots. Serialize adoption against + // a producer's close -> quarantine rename -> fresh-slot recreate + // transition, then revalidate while that stable parent-anchored + // lock is held. The slot's own .lock inode moves with a rename and + // cannot provide this guarantee by itself. + if (slotPath != null) { + try { + logicalSlotLock = SlotLock.acquireLogical(slotPath); + } catch (IllegalStateException t) { + if (isLockContention(t)) { + LOG.info("orphan logical slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } + throw t; + } + if (!OrphanScanner.isCandidateOrphan(slotPath)) { + LOG.info("orphan candidate changed before adoption, skipping: {}", slotPath); + outcome = DrainOutcome.SUCCESS; + return; + } + } + + // The engine acquires the directory-local .lock itself. Keep the + // lock order logical -> local, and release the short-lived logical + // lock only after the engine has secured stable ownership. try { engine = new CursorSendEngine(slotPath, segmentSizeBytes, sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); } catch (IllegalStateException t) { - String msg = t.getMessage(); - if (msg != null && msg.contains("already in use")) { + if (isLockContention(t)) { LOG.info("orphan slot already locked, skipping: {} ({})", - slotPath, msg); + slotPath, t.getMessage()); outcome = DrainOutcome.LOCKED_BY_OTHER; return; } throw t; } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + logicalSlotLock = null; + } long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", @@ -724,12 +753,20 @@ public void run() { + "slot lock releases when it exits", slotPath); } } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + } // Don't let a later requestStop() unpark an unrelated task that // the pool's executor may have scheduled onto this same thread. runnerThread = null; } } + private static boolean isLockContention(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.contains("already in use"); + } + /** * Plug an observer for durable-ack-related events. {@code null} clears * any previously installed listener. See {@link BackgroundDrainerListener} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 0b8379de..eb6281a7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -33,10 +33,14 @@ import java.nio.charset.StandardCharsets; /** - * Advisory exclusive lock for a single SF slot directory. + * Advisory exclusive locks for a single SF slot. *

      - * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx} - * for the entire lifetime of the engine that owns the slot. The lock is + * {@link #acquire(String)} locks a {@code .lock} file inside the slot directory + * for the entire lifetime of the engine that owns it. {@link #acquireLogical(String)} + * locks a sibling file under the parent SF directory for short-lived pathname + * transitions and orphan adoption; because it is outside the slot directory, + * it remains stable if that directory is renamed. Both use + * {@code flock}/{@code LockFileEx}. A lock is * automatically released when the fd is closed — including on hard process * exit, since the kernel cleans up file locks for terminated processes. *

      @@ -58,6 +62,7 @@ public final class SlotLock implements QuietCloseable { private static final String LOCK_FILE_NAME = ".lock"; private static final String LOCK_PID_FILE_NAME = ".lock.pid"; + private static final String LOGICAL_LOCK_DIR_NAME = ".slot-locks"; private final String slotDir; private int fd; @@ -75,18 +80,42 @@ private SlotLock(String slotDir, int fd) { * or lock contention. */ public static SlotLock acquire(String slotDir) { - if (slotDir == null || slotDir.isEmpty()) { - throw new IllegalArgumentException("slotDir must not be empty"); - } - if (!Files.exists(slotDir)) { - int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT); - if (rc != 0) { - throw new IllegalStateException( - "could not create slot dir: " + slotDir + " rc=" + rc); - } - } + validateSlotDir(slotDir); + ensureDirectory(slotDir, "slot dir"); String lockPath = slotDir + "/" + LOCK_FILE_NAME; String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME; + return acquireAt(slotDir, lockPath, pidPath); + } + + /** + * Acquires the stable logical lock for {@code slotDir}. Unlike the + * directory-local {@code .lock}, this lock is anchored in the parent SF + * directory, so renaming the slot cannot move the lock inode away from the + * logical slot name it guards. + *

      + * Callers use this as a short-lived transition/adoption lock, always before + * acquiring the directory-local lock. In particular it must cover an + * unreplayable slot's close -> rename -> recreate transition, preventing a + * queued orphan drainer from adopting the renamed inode and later touching + * the fresh directory through the old pathname. + */ + public static SlotLock acquireLogical(String slotDir) { + validateSlotDir(slotDir); + int separator = slotDir.lastIndexOf('/'); + if (separator < 1 || separator == slotDir.length() - 1) { + throw new IllegalArgumentException( + "slotDir must contain a parent and slot name: " + slotDir); + } + String parentDir = slotDir.substring(0, separator); + String slotName = slotDir.substring(separator + 1); + String logicalLockDir = parentDir + "/" + LOGICAL_LOCK_DIR_NAME; + ensureDirectory(logicalLockDir, "logical slot lock dir"); + String lockPath = logicalLockDir + "/" + slotName + ".lock"; + String pidPath = logicalLockDir + "/" + slotName + ".lock.pid"; + return acquireAt(slotDir, lockPath, pidPath); + } + + private static SlotLock acquireAt(String slotDir, String lockPath, String pidPath) { int fd = Files.openRW(lockPath); if (fd < 0) { throw new IllegalStateException( @@ -111,6 +140,25 @@ public static SlotLock acquire(String slotDir) { } } + private static void ensureDirectory(String path, String description) { + if (!Files.exists(path)) { + int rc = Files.mkdir(path, Files.DIR_MODE_DEFAULT); + // Multiple senders may create the shared parent lock directory + // concurrently. Treat EEXIST as success, just as the builder does + // for the SF root itself. + if (rc != 0 && !Files.exists(path)) { + throw new IllegalStateException( + "could not create " + description + ": " + path + " rc=" + rc); + } + } + } + + private static void validateSlotDir(String slotDir) { + if (slotDir == null || slotDir.isEmpty()) { + throw new IllegalArgumentException("slotDir must not be empty"); + } + } + /** Slot dir this lock guards. */ public String slotDir() { return slotDir; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index b4d91895..d14c97a8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -27,10 +27,13 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; -import io.questdb.client.test.tools.DelegatingFilesFacade; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.std.ObjList; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Assert; @@ -46,9 +49,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -249,6 +254,103 @@ public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() th }); } + @Test + public void testQueuedOrphanCannotAdoptSlotWhileQuarantineRecreatesItsName() throws Exception { + // Sender 1 leaves a genuinely unreplayable default slot. Sender 2 recovers it, + // closes the directory-local lock, quarantines the directory and recreates the + // default pathname. A scanner may already have queued that pathname before sender + // 2 starts; the queued drainer must not acquire the old lock inode in the close -> + // rename gap and later issue path-based writes/unlinks against sender 2's fresh slot. + assertMemoryLeak(() -> { + writeAndTearUnreplayableSlot(); + ObjList scannerSnapshot = OrphanScanner.scan(sfDir, "other-sender"); + Assert.assertEquals(1, scannerSnapshot.size()); + String staleSnapshotPath = scannerSnapshot.get(0); + + CountDownLatch producerInRenameGap = new CountDownLatch(1); + CountDownLatch allowQuarantineRename = new CountDownLatch(1); + AtomicReference recoveredSender = new AtomicReference<>(); + AtomicReference recoveryFailure = new AtomicReference<>(); + Thread recoveryThread = null; + Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest(() -> { + producerInRenameGap.countDown(); + try { + if (!allowQuarantineRename.await(15, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to finish quarantine rename"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("quarantine hook interrupted", e); + } + }); + try { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + + recoveryThread = new Thread(() -> { + try { + recoveredSender.set(Sender.fromConfig(cfg)); + } catch (Throwable t) { + recoveryFailure.set(t); + } + }, "qwp-quarantine-recovery"); + recoveryThread.start(); + Assert.assertTrue("recovering sender did not reach the quarantine gap", + producerInRenameGap.await(10, TimeUnit.SECONDS)); + + BackgroundDrainer queuedDrainer = new BackgroundDrainer( + staleSnapshotPath, 256, 8192, () -> null, + 1000, 1, 10, true, 0); + Thread drainerThread = new Thread(queuedDrainer, "qwp-queued-orphan"); + drainerThread.start(); + drainerThread.join(5_000); + Assert.assertFalse("queued drainer must not wait on or adopt the old inode", + drainerThread.isAlive()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.LOCKED_BY_OTHER, + queuedDrainer.outcome()); + + allowQuarantineRename.countDown(); + recoveryThread.join(10_000); + Assert.assertFalse("recovering sender did not finish", recoveryThread.isAlive()); + if (recoveryFailure.get() != null) { + throw new AssertionError("recovering sender failed", recoveryFailure.get()); + } + Sender sender = recoveredSender.get(); + Assert.assertNotNull(sender); + sender.table("m").symbol("s", "after-race").longColumn("v", 99).atNow(); + sender.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { + Thread.sleep(20); + } + sender.close(); + recoveredSender.set(null); + + assertUnreplayableSlotSetAside(); + Assert.assertEquals("fresh producer slot must remain intact and usable", + Arrays.asList("after-race"), handler.dictSnapshot()); + } + } finally { + allowQuarantineRename.countDown(); + if (recoveryThread != null) { + recoveryThread.join(10_000); + if (recoveryThread.isAlive()) { + recoveryThread.interrupt(); + } + } + Sender sender = recoveredSender.getAndSet(null); + if (sender != null) { + sender.close(); + } + Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest(null); + } + }); + } + @Test public void testFullyAckedTornSlotResumesInPlaceWithoutQuarantine() throws Exception { // M1 regression -- the ACKED counterpart to diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java index 162474fe..a39bef67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.std.Files; import io.questdb.client.std.IntList; @@ -321,6 +322,30 @@ public void testIsCandidateOrphanDirect() throws Exception { }); } + @Test + public void testDrainerRevalidatesStaleScannerSnapshotBeforeCreatingEngine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = sfDir + "/stale"; + assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + String segment = slot + "/sf-0001.sfa"; + touchFile(segment); + assertTrue(OrphanScanner.isCandidateOrphan(slot)); + + // The scanner has already queued this pathname, but another owner + // drains/removes its last segment before the worker gets to run. + assertTrue(Files.remove(segment)); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, 1024, 8192, () -> { + throw new AssertionError("a stale candidate must not connect"); + }, 1000, 1, 10, true, 0); + drainer.run(); + + assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + assertFalse("stale adoption must not create the directory-local lock", + Files.exists(slot + "/.lock")); + }); + } + private static void touchFile(String path) { int fd = Files.openRW(path); if (fd >= 0) Files.close(fd); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 644a7463..6643bff8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -103,6 +103,30 @@ public void testCloseReleasesLock() throws Exception { }); } + @Test + public void testLogicalLockRemainsContendedAcrossSlotRenameAndRecreate() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/rename"; + String moved = parentDir + "/rename.quarantined"; + assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + + try (SlotLock ignored = SlotLock.acquireLogical(slot)) { + assertEquals(0, Files.rename(slot, moved)); + assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + + try (SlotLock unexpected = SlotLock.acquireLogical(slot)) { + fail("logical slot lock must survive rename and recreate"); + } catch (IllegalStateException expected) { + assertTrue(expected.getMessage().contains("already in use")); + } + } + + try (SlotLock reacquired = SlotLock.acquireLogical(slot)) { + assertEquals(slot, reacquired.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { From 6328467c85add2f5823612d59ae4c60ee8d433b1 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:52:14 +0100 Subject: [PATCH 69/78] Verify delivery after torn-slot recovery --- .../client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index d14c97a8..f9f0e968 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -394,6 +394,10 @@ public void testFullyAckedTornSlotResumesInPlaceWithoutQuarantine() throws Excep while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { Thread.sleep(20); } + Assert.assertEquals( + "the resumed sender must deliver new data from the original slot", + Arrays.asList("after-recovery"), + handler.dictSnapshot()); } } // The fully-acked slot was NOT quarantined: no set-aside copy exists (the From 5617d7906775cfe21c5befde7557637ac8dd8259 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 22:21:06 +0100 Subject: [PATCH 70/78] Optimize symbol dictionary recovery and catch-up --- .../client/sf/cursor/CursorSendEngine.java | 84 ++++-- .../sf/cursor/CursorWebSocketSendLoop.java | 71 ++++- .../qwp/client/sf/cursor/MmapSegment.java | 14 + .../client/sf/cursor/PersistedSymbolDict.java | 189 ++++++++++++- .../sf/cursor/RecoveredFrameAnalysis.java | 261 ++++++++++++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 18 ++ ...WebSocketSendLoopCatchUpAlignmentTest.java | 32 +++ ...CursorWebSocketSendLoopOrphanTailTest.java | 52 ++++ .../sf/cursor/PersistedSymbolDictTest.java | 34 +++ 9 files changed, 716 insertions(+), 39 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 6b1d741a..5fdc129c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -137,6 +137,10 @@ public final class CursorSendEngine implements QuietCloseable { // constructor, closed by {@link #close()}. When null in disk mode the engine // reports delta encoding as unavailable and the sender keeps full-dict frames. private final PersistedSymbolDict persistedSymbolDict; + // Engine-owned output of the single ordered recovery walk. It is retained + // because both producer seeding and every recycled send loop need the same + // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. + private final RecoveredFrameAnalysis recoveredFrameAnalysis; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -248,6 +252,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; PersistedSymbolDict persistedDictInProgress = null; + RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine @@ -329,6 +334,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (seed >= 0) { recovered.acknowledge(seed); } + // Fold the whole recovered ring once. The result checkpoints all + // running metadata and raw symbol bytes at each commit-bearing + // frame, so its final snapshot excludes an orphan deferred tail + // without requiring a second bounded scan. + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery( + persistedDictInProgress == null ? 0 : persistedDictInProgress.size()); // Locate the last commit-bearing frame below a potentially // orphaned FLAG_DEFER_COMMIT tail. A producer that crashed (or // closed) mid-transaction leaves deferred frames with no @@ -338,12 +349,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // drainOnClose), and (b) replaying them into a NEW session's // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. - this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - QwpConstants.HEADER_OFFSET_FLAGS, - QwpConstants.FLAG_DEFER_COMMIT, - QwpConstants.MAGIC_MESSAGE, - QwpConstants.HEADER_SIZE - ); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; LOG.warn("recovered SF log ends with {} deferred frame(s) whose transaction was never " @@ -363,12 +369,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // this and over-reject an otherwise-recoverable slot. maxSymbolDeltaEnd // returns 0 when no such frame carries a symbol, yielding -1 here. // Computed before the I/O loop or producer append; single-threaded. - this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( - QwpConstants.MAGIC_MESSAGE, - QwpConstants.HEADER_OFFSET_FLAGS, - QwpConstants.FLAG_DELTA_SYMBOL_DICT, - QwpConstants.HEADER_SIZE, - recoveredCommitBoundaryFsn) - 1L; + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; // Full-dict-fallback recovery. When the persisted .symbol-dict is a // SUBSET of the ids the surviving frames reference // (recoveredMaxSymbolId >= its size) YET every such frame is @@ -388,12 +389,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // dictionary. The recoveredMaxSymbolId >= size guard means this never // fires for a slot whose dictionary is intact, nor for an empty slot // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. - this.recoveredMaxSymbolDeltaStart = recovered.maxSymbolDeltaStart( - QwpConstants.MAGIC_MESSAGE, - QwpConstants.HEADER_OFFSET_FLAGS, - QwpConstants.FLAG_DELTA_SYMBOL_DICT, - QwpConstants.HEADER_SIZE, - recoveredCommitBoundaryFsn); + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); if (persistedDictInProgress != null && recoveredMaxSymbolId >= persistedDictInProgress.size() && recoveredMaxSymbolDeltaStart == 0L) { @@ -451,6 +447,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.ring = ringInProgress; this.watermark = watermarkInProgress; this.persistedSymbolDict = persistedDictInProgress; + this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -482,6 +479,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (recoveredFrameAnalysisInProgress != null) { + try { + recoveredFrameAnalysisInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -667,6 +670,12 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (recoveredFrameAnalysis != null) { + try { + recoveredFrameAnalysis.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -716,6 +725,11 @@ public synchronized void close() { * transmitted, so their ids never reach a server and must not inflate the baseline. */ public long collectReplaySymbolsAbove(int baseline, ObjList out) { + if (recoveredFrameAnalysis != null + && recoveredFrameAnalysis.baseline() == baseline) { + recoveredFrameAnalysis.appendDecodedSymbols(out); + return recoveredFrameAnalysis.coverage(); + } if (ring == null) { return baseline; } @@ -738,6 +752,40 @@ public long collectReplaySymbolsAbove(int baseline, ObjList out) { ); } + long recoveredSymbolCoverage(int baseline) { + return checkedRecoveryAnalysis(baseline).coverage(); + } + + int recoveredSymbolSuffixCount(int baseline) { + return checkedRecoveryAnalysis(baseline).rawCount(); + } + + int recoveredSymbolSuffixLen(int baseline) { + return checkedRecoveryAnalysis(baseline).rawLen(); + } + + void copyRecoveredSymbolSuffix(int baseline, long target) { + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + int len = analysis.rawLen(); + if (len > 0) { + io.questdb.client.std.Unsafe.getUnsafe().copyMemory(analysis.rawAddr(), target, len); + } + } + + @TestOnly + public long recoveryFramesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited(); + } + + private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) { + if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) { + throw new IllegalStateException("recovery symbol baseline mismatch [expected=" + + (recoveredFrameAnalysis == null ? "none" : recoveredFrameAnalysis.baseline()) + + ", actual=" + baseline + ']'); + } + return recoveredFrameAnalysis; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 441edd90..5c59823c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -44,7 +44,6 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.str.Utf8s; -import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; @@ -195,7 +194,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; - // Test seam: when true, appendSymbolToMirror throws instead of appending, + // Test seam: when true, recovery mirror seeding throws before growing, // simulating the native realloc OOM (or MAX_SENT_DICT_BYTES ceiling) that // ensureSentDictCapacity can raise while the constructor seeds the recovery // mirror. Lets a test prove the constructor frees the already-malloc'd mirror on @@ -300,6 +299,13 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private int sentDictBytesCapacity; private int sentDictBytesLen; private int sentDictCount; + // Reusable staging frame for dictionary catch-up chunks. A reconnect may split a + // large dictionary into many chunks, and every later reconnect repeats that split; + // retaining one growable loop-owned buffer turns the old malloc/free-per-chunk path + // into amortized growth only. I/O-thread-owned, freed beside sentDictBytesAddr. + private long catchUpFrameAddr; + private int catchUpFrameCapacity; + private int catchUpFrameGrowthCount; // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry // too large for its batch cap -- with no intervening successful catch-up or unrelated // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). @@ -750,18 +756,28 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // catch-up-free. if (engine.recoveredMaxSymbolDeltaStart() > 0L) { // The prefix seed above may already have malloc'd the mirror, and - // appendSymbolToMirror -> ensureSentDictCapacity can still throw here (a - // native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling). Such a throw + // ensureSentDictCapacity can still throw here (a native realloc OOM, or + // the MAX_SENT_DICT_BYTES ceiling). Such a throw // propagates out of the constructor, so the half-built loop is never // assigned to a reference -- neither ensureConnected's catch nor // BackgroundDrainer's finally can close() it -- and the mirror would leak. // Free it on any throw so the constructor leaves nothing behind, mirroring // the loopNeverRan free in close() / ioLoop's exit. try { - ObjList fromFrames = new ObjList<>(); - if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { - for (int i = 0, n = fromFrames.size(); i < n; i++) { - appendSymbolToMirror(fromFrames.getQuick(i)); + int baseline = sentDictCount; + if (engine.recoveredSymbolCoverage(baseline) >= 0L) { + int suffixLen = engine.recoveredSymbolSuffixLen(baseline); + int suffixCount = engine.recoveredSymbolSuffixCount(baseline); + if (suffixLen > 0) { + if (forceMirrorSeedFailureForTest) { + throw new LineSenderException( + "simulated mirror seed allocation failure (test only)"); + } + ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); + engine.copyRecoveredSymbolSuffix( + baseline, sentDictBytesAddr + sentDictBytesLen); + sentDictBytesLen += suffixLen; + sentDictCount += suffixCount; } } } catch (Throwable t) { @@ -1120,6 +1136,9 @@ public synchronized void close() { // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. sentDictCount = 0; } + if (loopNeverRan) { + freeCatchUpFrameBuffer(); + } } /** @@ -2030,6 +2049,7 @@ private void ioLoop() { sentDictBytesLen = 0; sentDictCount = 0; // keep the mirror all-or-nothing (see close()) } + freeCatchUpFrameBuffer(); shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -2613,7 +2633,8 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, } int payloadLen = (int) payloadLenL; int frameLen = (int) frameLenL; - long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); + ensureCatchUpFrameCapacity(frameLen); + long frame = catchUpFrameAddr; try { Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); @@ -2647,14 +2668,42 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, throw (Error) t; } throw new CatchUpSendException(t); - } finally { - Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); } nextWireSeq++; // this catch-up chunk consumed a wire sequence lastFrameOrPingNanos = System.nanoTime(); totalFramesSent.incrementAndGet(); } + @TestOnly + public int catchUpFrameGrowthCount() { + return catchUpFrameGrowthCount; + } + + private void ensureCatchUpFrameCapacity(int required) { + if (catchUpFrameCapacity >= required) { + return; + } + long newCapacity = Math.max(required, Math.max(4096L, (long) catchUpFrameCapacity * 2L)); + if (newCapacity > MAX_SENT_DICT_BYTES) { + newCapacity = MAX_SENT_DICT_BYTES; + } + catchUpFrameAddr = Unsafe.realloc( + catchUpFrameAddr, + catchUpFrameCapacity, + (int) newCapacity, + MemoryTag.NATIVE_DEFAULT); + catchUpFrameCapacity = (int) newCapacity; + catchUpFrameGrowthCount++; + } + + private void freeCatchUpFrameBuffer() { + if (catchUpFrameAddr != 0L) { + Unsafe.free(catchUpFrameAddr, catchUpFrameCapacity, MemoryTag.NATIVE_DEFAULT); + catchUpFrameAddr = 0L; + catchUpFrameCapacity = 0; + } + } + private boolean tryReceiveAcks() { boolean any = false; try { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index b2ee674b..8942cdfb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -639,6 +639,20 @@ public long collectReplaySymbolsAbove( return coverage; } + /** + * Feeds every recovered frame in this segment to the engine's single-pass + * recovery fold. Segments are visited oldest-first by {@link SegmentRing}. + */ + void scanRecovery(RecoveredFrameAnalysis analysis) { + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + analysis.accept(baseSeq + i, mmapAddress + off + FRAME_HEADER_SIZE, payloadLen); + off += FRAME_HEADER_SIZE + payloadLen; + } + } + /** * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any * delta-flagged frame in this segment carries, or {@code 0} only when NO diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 835d365a..982f93c2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Crc32c; +import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -130,9 +131,9 @@ * owner (the engine) closes it, and {@code close()} is callable from any thread * (a shutdown hook, test cleanup). {@code close()} and the append methods are * therefore {@code synchronized}: without that, a close racing an in-flight append - * could free the scratch buffer or close the fd mid-write and let the write land - * on a descriptor the OS has reused for another file (silent cross-file - * corruption). Not thread-safe for concurrent writers. + * could unmap the production append region (or free the fault-test scratch buffer), + * close the fd, and let an in-flight write corrupt memory or land on a descriptor + * the OS has reused for another file. Not thread-safe for concurrent writers. */ public final class PersistedSymbolDict implements QuietCloseable { @@ -145,6 +146,7 @@ public final class PersistedSymbolDict implements QuietCloseable { static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; + static final int INITIAL_APPEND_MAP_CAPACITY = 64 * 1024; /** * Upper bound on a chunk's two header varints ({@code entryCount} and * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The @@ -167,7 +169,15 @@ public final class PersistedSymbolDict implements QuietCloseable { // tests inject a fault facade to exercise recovery I/O failures (a truncate // that cannot drop a torn tail, a short write) without a real broken disk. private final FilesFacade ff; + // Production writes directly into segmented append mappings. Custom facades retain the + // positioned-write path so fault tests can inject short writes through ff.write. + private final boolean mappedAppend; + private long appendMapAddr; + private long appendMapCapacity; + private long appendMapOffset; + private int appendMapGrowthCount; private long appendOffset; + private long appendWriteCount; private boolean closed; // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and // CRCs stripped -- populated only when open() recovered existing chunks @@ -186,6 +196,7 @@ public final class PersistedSymbolDict implements QuietCloseable { private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { this.ff = ff; this.fd = fd; + this.mappedAppend = ff == FilesFacade.INSTANCE; this.appendOffset = appendOffset; this.size = size; this.loadedEntriesAddr = loadedEntriesAddr; @@ -332,6 +343,16 @@ public synchronized void appendRawEntries(long addr, int len, int count) { // production. assert validateRawEntries(addr, len, count); int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len); + if (mappedAppend) { + long recLen = (long) hdrLen + len + CRC_SIZE; + ensureAppendMap(checkedRequiredOffset(recLen)); + long recStart = appendMapAddr + appendOffset - appendMapOffset; + long p = NativeBufferWriter.writeVarint(recStart, count); + NativeBufferWriter.writeVarint(p, len); + Unsafe.getUnsafe().copyMemory(addr, recStart + hdrLen, len); + commitMappedChunk(recStart, hdrLen, len, count); + return; + } ensureScratch((long) hdrLen + len + CRC_SIZE); long p = NativeBufferWriter.writeVarint(scratchAddr, count); NativeBufferWriter.writeVarint(p, len); @@ -357,6 +378,20 @@ public synchronized void appendSymbol(CharSequence symbol) { } int utf8Len = Utf8s.utf8Bytes(symbol); int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + if (mappedAppend) { + int hdrLen = NativeBufferWriter.varintSize(1) + NativeBufferWriter.varintSize(wireLen); + long recLen = (long) hdrLen + wireLen + CRC_SIZE; + ensureAppendMap(checkedRequiredOffset(recLen)); + long recStart = appendMapAddr + appendOffset - appendMapOffset; + long p = NativeBufferWriter.writeVarint(recStart, 1); + p = NativeBufferWriter.writeVarint(p, wireLen); + p = NativeBufferWriter.writeVarint(p, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + commitMappedChunk(recStart, hdrLen, wireLen, 1); + return; + } ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE); long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE; long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); @@ -367,13 +402,12 @@ public synchronized void appendSymbol(CharSequence symbol) { } /** - * Appends the dense id range {@code [from .. to]} as ONE chunk, in a SINGLE - * write with a SINGLE checksum. Encodes the whole entry region into scratch - * first -- versus one {@code pwrite(2)} and one native CRC call per symbol. - * That per-symbol syscall and JNI count is the hot-path cost on a - * high-cardinality batch (one new symbol per row), which is exactly the - * store-and-forward workload delta encoding targets. Callers pass the dictionary - * and the range so the ids resolve to their symbol strings. + * Appends the dense id range {@code [from .. to]} as one chunk with one + * checksum. Production encodes directly into a segmented append mapping, avoiding + * both the staging copy and a positioned-write syscall on every flush. The + * scratch/positioned-write fallback exists only behind an injected filesystem + * facade so short-write recovery tests retain their fault seam. Callers pass the + * dictionary and the range so the ids resolve to their symbol strings. *

      * Same durability and idempotency contract as {@link #appendSymbol}: no fsync, * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset}, @@ -385,6 +419,37 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in return; } int count = to - from + 1; + if (mappedAppend) { + long entriesLenLong = 0L; + for (int id = from; id <= to; id++) { + int utf8Len = Utf8s.utf8Bytes(dict.getSymbol(id)); + entriesLenLong += NativeBufferWriter.varintSize(utf8Len) + (long) utf8Len; + if (entriesLenLong > MAX_SCRATCH_BYTES) { + throw new IllegalStateException("symbol dict chunk exceeds the maximum size to " + + FILE_NAME + " [required=" + entriesLenLong + ", max=" + + MAX_SCRATCH_BYTES + ']'); + } + } + int entriesLen = (int) entriesLenLong; + int hdrLen = NativeBufferWriter.varintSize(count) + + NativeBufferWriter.varintSize(entriesLen); + long recLen = (long) hdrLen + entriesLen + CRC_SIZE; + ensureAppendMap(checkedRequiredOffset(recLen)); + long recStart = appendMapAddr + appendOffset - appendMapOffset; + long p = NativeBufferWriter.writeVarint(recStart, count); + p = NativeBufferWriter.writeVarint(p, entriesLen); + for (int id = from; id <= to; id++) { + CharSequence symbol = dict.getSymbol(id); + int utf8Len = Utf8s.utf8Bytes(symbol); + p = NativeBufferWriter.writeVarint(p, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + p += utf8Len; + } + } + commitMappedChunk(recStart, hdrLen, entriesLen, count); + return; + } int entriesLen = 0; for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); @@ -415,6 +480,20 @@ public synchronized void close() { loadedEntriesAddr = 0L; loadedEntriesLen = 0; } + if (appendMapAddr != 0L) { + Files.munmap(appendMapAddr, appendMapCapacity, MemoryTag.MMAP_DEFAULT); + appendMapAddr = 0L; + appendMapCapacity = 0L; + appendMapOffset = 0L; + // The active window reserves space past the logical end. Return that tail on + // orderly close; after a crash open() treats the zero-filled reserve as + // a torn trailing chunk and truncates it to the same appendOffset. + if (!ff.truncate(fd, appendOffset)) { + LOG.warn("symbol dict {} could not trim mmap reserve to {}; recovery will " + + "discard the zero-filled tail on the next open", + FILE_NAME, appendOffset); + } + } if (scratchAddr != 0L) { Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); scratchAddr = 0L; @@ -425,6 +504,16 @@ public synchronized void close() { } } + @TestOnly + public synchronized int appendMapGrowthCount() { + return appendMapGrowthCount; + } + + @TestOnly + public synchronized long appendWriteCount() { + return appendWriteCount; + } + /** * Base address of the loaded entry region -- the concatenated * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a @@ -726,6 +815,85 @@ private static boolean validateRawEntries(long addr, int len, int count) { return true; } + private long checkedRequiredOffset(long recordLen) { + long required = appendOffset + recordLen; + if (recordLen < 0L || required < appendOffset) { + throw new IllegalStateException("symbol dict append offset overflow to " + + FILE_NAME + " [offset=" + appendOffset + ", recordLen=" + recordLen + ']'); + } + return required; + } + + /** + * Commits a chunk already assembled directly in the append mmap. The logical + * offset and symbol count advance last, after the CRC and a store fence, so an + * interrupted process leaves either a complete chunk or a tail that open() + * rejects and truncates. + */ + private void commitMappedChunk(long recStart, int hdrLen, int entriesLen, int count) { + long bodyLen = (long) hdrLen + entriesLen; + long recLen = bodyLen + CRC_SIZE; + Unsafe.getUnsafe().putInt( + recStart + bodyLen, + Crc32c.update(Crc32c.INIT, recStart, bodyLen)); + Unsafe.getUnsafe().storeFence(); + appendOffset += recLen; + size += count; + } + + /** + * Ensures the production append mmap covers the absolute file offset + * {@code required}. The log is mapped in 64 KiB-aligned windows: small flushes + * share a window, while a large existing dictionary does not force the process + * to reserve and remap its whole prefix (or geometrically over-allocate it). + */ + private void ensureAppendMap(long required) { + if (appendMapAddr != 0L + && required >= appendMapOffset + && required <= appendMapOffset + appendMapCapacity) { + return; + } + long newOffset = appendOffset - appendOffset % INITIAL_APPEND_MAP_CAPACITY; + long needed = required - newOffset; + long newCapacity = Math.max(INITIAL_APPEND_MAP_CAPACITY, needed); + long remainder = newCapacity % INITIAL_APPEND_MAP_CAPACITY; + if (remainder != 0L) { + long padding = INITIAL_APPEND_MAP_CAPACITY - remainder; + if (newCapacity > Long.MAX_VALUE - padding) { + throw new IllegalStateException("symbol dict mmap capacity overflow to " + + FILE_NAME + " [required=" + required + ']'); + } + newCapacity += padding; + } + if (newOffset > Long.MAX_VALUE - newCapacity) { + throw new IllegalStateException("symbol dict mmap file offset overflow to " + + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']'); + } + long newFileSize = newOffset + newCapacity; + if (!ff.allocate(fd, newFileSize)) { + throw new IllegalStateException("could not grow mmap append region for " + + FILE_NAME + " [required=" + required + ", fileSize=" + newFileSize + ']'); + } + if (appendMapAddr != 0L) { + long oldAddr = appendMapAddr; + long oldCapacity = appendMapCapacity; + appendMapAddr = 0L; + appendMapCapacity = 0L; + appendMapOffset = 0L; + Files.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT); + } + long newAddr = Files.mmap( + fd, newCapacity, newOffset, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + if (newAddr == Files.FAILED_MMAP_ADDRESS) { + throw new IllegalStateException("could not mmap append region for " + + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']'); + } + appendMapAddr = newAddr; + appendMapCapacity = newCapacity; + appendMapOffset = newOffset; + appendMapGrowthCount++; + } + /** * Grows the append scratch buffer to hold at least {@code required} bytes. *

      @@ -769,6 +937,7 @@ private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) { int bodyLen = hdrLen + entriesLen; int recLen = bodyLen + CRC_SIZE; Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen)); + appendWriteCount++; long written = ff.write(fd, recStart, recLen, appendOffset); if (written != recLen) { throw new IllegalStateException("short write to " + FILE_NAME diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java new file mode 100644 index 00000000..c9650cfd --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -0,0 +1,261 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; + +/** + * Engine-owned result of the one ordered frame walk performed during recovery. + * It folds commit-boundary detection, delta extrema, gap detection and the raw + * symbol suffix into the same pass. Running state is checkpointed only at a + * commit-bearing frame, so a deferred orphan tail can be scanned without ever + * leaking its metadata or symbols into the committed result. + */ +final class RecoveredFrameAnalysis implements QuietCloseable { + + private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8; + private final int baseline; + private long committedBoundaryFsn = -1L; + private long committedCoverage; + private boolean committedGap; + private long committedMaxDeltaEnd; + private long committedMaxDeltaStart; + private int committedRawCount; + private int committedRawLen; + private long framesVisited; + private boolean runningGap; + private long runningMaxDeltaEnd; + private long runningMaxDeltaStart; + private long rawAddr; + private int rawCapacity; + private int runningRawCount; + private int runningRawLen; + private long runningCoverage; + + RecoveredFrameAnalysis(int baseline) { + this.baseline = baseline; + this.runningCoverage = baseline; + this.committedCoverage = baseline; + } + + void accept(long fsn, long payload, int payloadLen) { + framesVisited++; + boolean isQwp = payloadLen >= QwpConstants.HEADER_SIZE + && payloadLen > QwpConstants.HEADER_OFFSET_FLAGS + && Unsafe.getUnsafe().getInt(payload) == QwpConstants.MAGIC_MESSAGE; + byte flags = isQwp + ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS) + : 0; + if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) { + foldDelta(payload + QwpConstants.HEADER_SIZE, payload + payloadLen); + } + + // Only a positively identified deferred QWP frame can belong to an + // uncommitted tail. Short, foreign or otherwise non-QWP payloads remain + // retirement barriers, matching findLastFrameFsnWithoutPayloadFlag. + if (!isQwp || (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0) { + committedBoundaryFsn = fsn; + committedCoverage = runningCoverage; + committedGap = runningGap; + committedMaxDeltaEnd = runningMaxDeltaEnd; + committedMaxDeltaStart = runningMaxDeltaStart; + committedRawLen = runningRawLen; + committedRawCount = runningRawCount; + } + } + + void appendDecodedSymbols(ObjList out) { + long p = rawAddr; + long limit = rawAddr + committedRawLen; + for (int i = 0; i < committedRawCount; i++) { + long encoded = readVarint(p, limit); + if (encoded < 0L) { + throw new IllegalStateException("malformed cached symbol dictionary suffix"); + } + int varintLen = (int) (encoded & 7L); + long symbolLen = encoded >>> 3; + p += varintLen; + if (symbolLen > limit - p) { + throw new IllegalStateException("truncated cached symbol dictionary suffix"); + } + out.add(Utf8s.stringFromUtf8Bytes(p, p + symbolLen)); + p += symbolLen; + } + if (p != limit) { + throw new IllegalStateException("overfilled cached symbol dictionary suffix"); + } + } + + int baseline() { + return baseline; + } + + long commitBoundaryFsn() { + return committedBoundaryFsn; + } + + long coverage() { + return committedGap ? -1L : committedCoverage; + } + + long framesVisited() { + return framesVisited; + } + + long maxDeltaEnd() { + return committedMaxDeltaEnd; + } + + long maxDeltaStart() { + return committedMaxDeltaStart; + } + + long rawAddr() { + return rawAddr; + } + + int rawCount() { + return committedRawCount; + } + + int rawLen() { + return committedRawLen; + } + + @Override + public void close() { + if (rawAddr != 0L) { + Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT); + rawAddr = 0L; + rawCapacity = 0; + runningRawLen = 0; + committedRawLen = 0; + } + } + + private void appendRaw(long entryStart, int entryLen) { + long required = (long) runningRawLen + entryLen; + if (required > MAX_RAW_BYTES) { + throw new IllegalStateException("recovered symbol dictionary suffix exceeds maximum size " + + "[required=" + required + ", max=" + MAX_RAW_BYTES + ']'); + } + if (required > rawCapacity) { + long newCapacity = Math.max(required, Math.max(4_096L, (long) rawCapacity * 2L)); + if (newCapacity > MAX_RAW_BYTES) { + newCapacity = MAX_RAW_BYTES; + } + rawAddr = Unsafe.realloc(rawAddr, rawCapacity, (int) newCapacity, MemoryTag.NATIVE_DEFAULT); + rawCapacity = (int) newCapacity; + } + Unsafe.getUnsafe().copyMemory(entryStart, rawAddr + runningRawLen, entryLen); + runningRawLen += entryLen; + runningRawCount++; + } + + private void foldDelta(long p, long limit) { + long encodedStart = readVarint(p, limit); + if (encodedStart < 0L) { + runningGap = true; + return; + } + int startLen = (int) (encodedStart & 7L); + long deltaStart = encodedStart >>> 3; + p += startLen; + if (deltaStart > runningMaxDeltaStart) { + runningMaxDeltaStart = deltaStart; + } + + long encodedCount = readVarint(p, limit); + if (encodedCount < 0L) { + runningGap = true; + return; + } + int countLen = (int) (encodedCount & 7L); + long deltaCount = encodedCount >>> 3; + p += countLen; + long deltaEnd = deltaCount > Long.MAX_VALUE - deltaStart + ? Long.MAX_VALUE + : deltaStart + deltaCount; + if (deltaEnd > runningMaxDeltaEnd) { + runningMaxDeltaEnd = deltaEnd; + } + if (runningGap) { + return; + } + if (deltaStart > runningCoverage) { + runningGap = true; + return; + } + + long id = deltaStart; + for (long i = 0; i < deltaCount; i++, id++) { + long entryStart = p; + long encodedLen = readVarint(p, limit); + if (encodedLen < 0L) { + runningGap = true; + return; + } + int varintLen = (int) (encodedLen & 7L); + long symbolLen = encodedLen >>> 3; + p += varintLen; + if (symbolLen > limit - p) { + runningGap = true; + return; + } + p += symbolLen; + if (id >= runningCoverage) { + appendRaw(entryStart, (int) (p - entryStart)); + } + } + if (deltaEnd > runningCoverage) { + runningCoverage = deltaEnd; + } + } + + /** + * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an + * unterminated/oversized 32-bit protocol varint. + */ + private static long readVarint(long p, long limit) { + long value = 0L; + int shift = 0; + int bytes = 0; + while (p < limit && bytes < 5) { + byte b = Unsafe.getUnsafe().getByte(p++); + value |= (long) (b & 0x7F) << shift; + bytes++; + if ((b & 0x80) == 0) { + return (value << 3) | bytes; + } + shift += 7; + } + return -1L; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 06d1e64d..8e596a01 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -581,6 +581,24 @@ public synchronized long collectReplaySymbolsAbove( minFsnInclusive, maxFsnInclusive, coverage, out); } + /** + * Performs the one ordered recovery fold across sealed segments and the + * active segment. The returned native suffix remains owned by the caller. + */ + synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) { + RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline); + try { + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + sealedSegments.get(i).scanRecovery(analysis); + } + active.scanRecovery(analysis); + return analysis; + } catch (Throwable t) { + analysis.close(); + throw t; + } + } + /** * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below * {@code maxFsnInclusive} references (0 when none). See diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 43bd9972..34ef3c4a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -162,6 +162,38 @@ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Excepti }); } + @Test + public void testSplitCatchUpReusesOneFrameBufferAcrossReconnects() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(3_100); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 3_000), TestUtils.repeat("y", 3_000)); + + invokeSetWireBaselineWithCatchUp(loop, 0L); + assertEquals("the small cap must split the dictionary", 2, client.framesSent); + assertEquals("the split chunks fit the initial native buffer", + 1, loop.catchUpFrameGrowthCount()); + + client.cap = 7_000; + invokeSetWireBaselineWithCatchUp(loop, 0L); + assertEquals("the larger cap must combine the dictionary", 3, client.framesSent); + assertEquals("the combined frame must grow the retained native buffer once", + 2, loop.catchUpFrameGrowthCount()); + + invokeSetWireBaselineWithCatchUp(loop, 0L); + assertEquals("the next reconnect sends one combined frame", 4, client.framesSent); + assertEquals("the grown native frame buffer must be reused across reconnects", + 2, loop.catchUpFrameGrowthCount()); + } finally { + // assertMemoryLeak verifies that close releases the retained buffer. + loop.close(); + } + } + }); + } + @Test public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Exception { // A transient wire failure WHILE shipping the catch-up (the fresh diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index d19f0e12..52235637 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -33,6 +33,7 @@ import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -365,6 +366,38 @@ public void testZeroCountDeltaFrameAnchorsRecoveredMaxSymbolIdAtItsBaseline() th }); } + @Test + public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeltaSymbolFrame(engine, 0, 'a'); + appendDeltaSymbolFrame(engine, 1, 'b'); + appendDeltaSymbolFrame(engine, 2, 'c'); + } + + try (CursorSendEngine engine = newEngine()) { + assertEquals("one recovery visit per frame", 3L, engine.recoveryFramesVisited()); + + ObjList first = new ObjList<>(); + ObjList second = new ObjList<>(); + assertEquals(3L, engine.collectReplaySymbolsAbove(0, first)); + assertEquals(3L, engine.collectReplaySymbolsAbove(0, second)); + assertEquals("a", first.getQuick(0)); + assertEquals("c", second.getQuick(2)); + assertEquals("producer seed reads must reuse the recovery result", + 3L, engine.recoveryFramesVisited()); + + CursorWebSocketSendLoop loop = newLoop(engine, new ArrayList<>()); + try { + assertEquals("send-loop construction must copy the cached native suffix", + 3L, engine.recoveryFramesVisited()); + } finally { + loop.close(); + } + } + }); + } + // --------------------------------------------------------------------- // harness // --------------------------------------------------------------------- @@ -456,6 +489,25 @@ private static void appendDeltaFrame(CursorSendEngine engine, boolean defer, int } } + private static void appendDeltaSymbolFrame(CursorSendEngine engine, int deltaStart, char symbol) { + int size = HEADER_SIZE + 4; // start, count=1, symbolLen=1, symbol byte + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < size; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS, (byte) FLAG_DELTA_SYMBOL_DICT); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE, (byte) deltaStart); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 1, (byte) 1); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 2, (byte) 1); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 3, (byte) symbol); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException { long deadline = System.nanoTime() + 10_000_000_000L; while (engine.ackedFsn() < target) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 6c6dcf91..280155a7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -446,6 +446,40 @@ public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { }); } + @Test + public void testMappedAppendAmortizesFlushesWithoutPositionedWrites() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + for (int i = 0; i < 10_000; i++) { + d.appendSymbol("sym-" + i); + } + Assert.assertEquals(10_000, d.size()); + Assert.assertEquals("production appends must write directly into the mmap", + 0L, d.appendWriteCount()); + Assert.assertEquals("ten thousand flushes must require only three mapped windows", + 3, d.appendMapGrowthCount()); + } finally { + d.close(); + } + + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + Assert.assertEquals(10_000, re.size()); + Assert.assertEquals("sym-9999", re.readLoadedSymbols().getQuick(9_999)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + private static int varintSize(int v) { int n = 1; while ((v >>>= 7) != 0) { From f3282a49563c4ec479a07f16c6b34495638e146d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 22:30:57 +0100 Subject: [PATCH 71/78] Harden delta dictionary test correctness --- .../qwp/client/DeltaDictCatchUpTest.java | 51 ++++++++++++++++-- .../qwp/client/DeltaDictRecoveryTest.java | 54 ++++++++++++++++--- 2 files changed, 94 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index a7b2fb5f..21fdcd58 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -97,6 +97,7 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); Assert.assertEquals("alpha", conn2.get(0)); Assert.assertEquals("beta", conn2.get(1)); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -143,6 +144,7 @@ public void testReconnectPreservesMonotonicDeltaBaseline() throws Exception { + "baseline (deltaStart >= 1); a reset baseline re-ships the whole " + "dictionary from deltaStart 0", handler.conn2SawDeltaAboveBaseline); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -204,6 +206,7 @@ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exce Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); Assert.assertEquals(nearCapSymbol, conn2.get(0)); Assert.assertEquals("beta", conn2.get(1)); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -251,6 +254,7 @@ public void testForegroundCatchUpCapGapRetriesPastOrphanBudgetAndRecovers() thro Assert.assertTrue("foreground sender must recover and drain after the cap is restored", sender.awaitAckedFsn(targetFsn, 5_000)); } + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -307,10 +311,19 @@ public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Excep Assert.assertTrue("catch-up split into multiple frames (saw " + handler.zeroTableFramesOnConn2 + ")", handler.zeroTableFramesOnConn2 >= 2); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } + private static void assertAckSequencesStartAtZero(List ackSequenceStarts) { + Assert.assertFalse("the fixture must ACK at least one connection", ackSequenceStarts.isEmpty()); + for (int i = 0; i < ackSequenceStarts.size(); i++) { + Assert.assertEquals("connection " + (i + 1) + " must begin ACK sequencing at zero", + 0L, ackSequenceStarts.get(i).longValue()); + } + } + private static String symbolName(int i) { // 10-char symbols so 40 of them clearly exceed the advertised 160-byte cap. return "symbol" + (1000 + i); @@ -371,10 +384,15 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ // post-reconnect frame trips it. volatile boolean conn2SawDeltaAboveBaseline; volatile boolean sawZeroTableFrameOnConn2; + private final List ackSequenceStarts = new CopyOnWriteArrayList<>(); private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; private final AtomicLong nextSeq = new AtomicLong(0); + List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() // Copy under the lock: the caller iterates it unlocked while the @@ -390,6 +408,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien currentClient = client; connectionsAccepted.incrementAndGet(); dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + nextSeq.set(0); } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); @@ -414,7 +433,11 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } } try { - client.sendBinary(buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(buildAck(ackSequence)); // Drop the first connection right after ACKing its only frame, // forcing the sender to reconnect onto a fresh dictionary. if (connNumber == 1) { @@ -474,23 +497,33 @@ private static int tableCount(byte[] frame) { */ private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + private final List ackSequenceStarts = new CopyOnWriteArrayList<>(); private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; private volatile TestWebSocketServer server; + List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + void setServer(TestWebSocketServer server) { this.server = server; } @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { - if (currentClient != client) { + boolean newConnection = currentClient != client; + if (newConnection) { currentClient = client; connectionsAccepted.incrementAndGet(); nextSeq.set(0); } try { - client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(CatchUpHandler.buildAck(ackSequence)); if (connectionsAccepted.get() == 1) { // Connection 1 registered the big symbol. Shrink the cap so the // reconnect's catch-up budget can't fit it, then drop to force @@ -518,6 +551,7 @@ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocke // Set once the server has closed connection 1 (see CatchUpHandler.conn1Closed). volatile boolean conn1Closed; volatile int zeroTableFramesOnConn2; + private final List ackSequenceStarts = new CopyOnWriteArrayList<>(); private final List> dictsByConn = new CopyOnWriteArrayList<>(); private final int dropConn1AtDictSize; private final AtomicLong nextSeq = new AtomicLong(0); @@ -528,6 +562,10 @@ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocke this.dropConn1AtDictSize = dropConn1AtDictSize; } + List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() // Copy under the lock: the caller iterates it unlocked while the @@ -543,6 +581,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien currentClient = client; connectionsAccepted.incrementAndGet(); dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + nextSeq.set(0); } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); @@ -551,7 +590,11 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien zeroTableFramesOnConn2++; } try { - client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(CatchUpHandler.buildAck(ackSequence)); // Drop connection 1 only once it has learned the entire batch, so // the sender's sent-dictionary mirror is complete and the reconnect // catch-up must replay a dictionary larger than the batch cap. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index f9f0e968..b14b0708 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -789,8 +789,9 @@ public void testPersistFailureSurfacesAsLineSenderException() throws Exception { CursorSendEngine engine = new CursorSendEngine( slot, 4L * 1024 * 1024, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff); - try (Sender sender = QwpWebSocketSender.connect( - "localhost", port, null, 0, 0, 0L, null, false, engine)) { + Sender sender = QwpWebSocketSender.connect( + "localhost", port, null, 0, 0, 0L, null, false, engine); + try { ff.armed = true; // the next dictionary append short-writes sender.table("m").symbol("s", "boom").longColumn("v", 1L).atNow(); try { @@ -801,14 +802,42 @@ public void testPersistFailureSurfacesAsLineSenderException() throws Exception { + "not leak a raw IllegalStateException: " + expected.getMessage(), expected.getMessage().contains("failed to persist symbol dictionary before publish")); } - } catch (LineSenderException ignored) { - // close() re-flushes the still-buffered row; the facade has disarmed, so - // this normally succeeds. Either way it is not what we assert. + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered row; the facade has disarmed, so + // this normally succeeds. Either way it is not what we assert. + } } } }); } + @Test + public void testReconstructingFixtureResetsAckSequencePerConnection() throws Exception { + assertMemoryLeak(() -> { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + for (int i = 0; i < 2; i++) { + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue("connection " + (i + 1) + " must receive ACK zero", + sender.awaitAckedFsn(fsn, 5_000)); + } + } + + Assert.assertEquals("each fresh connection must restart wire ACK sequencing", + Arrays.asList(0L, 0L), handler.ackSequenceStarts()); + } + }); + } + @Test public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs @@ -1385,10 +1414,15 @@ private static int readVarint(byte[] buf, int[] pos) { */ private static class DictReconstructingHandler implements TestWebSocketServer.WebSocketServerHandler { volatile boolean sawCatchUpFrame; + private final List ackSequenceStarts = new ArrayList<>(); private final List dict = new ArrayList<>(); private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; + synchronized List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + synchronized List dictSnapshot() { return new ArrayList<>(dict); } @@ -1399,16 +1433,22 @@ synchronized int maxDictSize() { @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { - if (currentClient != client) { + boolean newConnection = currentClient != client; + if (newConnection) { currentClient = client; dict.clear(); // fresh server dictionary per connection + nextSeq.set(0); } accumulate(data); if (tableCount(data) == 0 && hasDelta(data)) { sawCatchUpFrame = true; } try { - client.sendBinary(buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(buildAck(ackSequence)); } catch (IOException e) { throw new RuntimeException(e); } From 675d036733ed9d86b988c6569b275d76e44bbdba Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 23:08:32 +0100 Subject: [PATCH 72/78] Avoid re-encoding split QWP frames --- .../qwp/client/QwpWebSocketEncoder.java | 108 +++++++++++++++++- .../qwp/client/QwpWebSocketSender.java | 96 ++++++++-------- .../qwp/client/QwpWebSocketEncoderTest.java | 96 +++++++++------- 3 files changed, 205 insertions(+), 95 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 8013330a..f38d9e93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; @@ -45,8 +47,10 @@ public class QwpWebSocketEncoder implements QuietCloseable { // .symbol-dict instead of re-encoding the same symbols (see // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaCount; private int deltaEntriesEnd; private int deltaEntriesStart; + private int deltaStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -73,8 +77,8 @@ public void beginMessage( int batchMaxId ) { buffer.reset(); - int deltaStart = confirmedMaxId + 1; - int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + deltaStart = confirmedMaxId + 1; + deltaCount = Math.max(0, batchMaxId - confirmedMaxId); byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); byte origFlags = flags; flags = headerFlags; @@ -100,6 +104,64 @@ public void close() { } } + /** + * Copies one single-table split message from the combined message currently + * staged in this encoder. The table body is copied byte-for-byte from its + * recorded offset; columns and rows are not encoded again. + */ + public int copySplitMessage( + MicrobatchBuffer target, + int tableBodyOffset, + int tableBodyLength, + boolean deferCommit, + int confirmedMaxId, + int batchMaxId + ) { + if (target.getBufferPos() != 0) { + throw new IllegalStateException("split message target is not empty"); + } + if (tableBodyOffset < deltaEntriesEnd + || tableBodyLength < 0 + || (long) tableBodyOffset + tableBodyLength > buffer.getPosition()) { + throw new IllegalArgumentException("table body slice is outside the staged message"); + } + + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + int messageSize = splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + target.ensureCapacity(messageSize); + + long source = buffer.getBufferPtr(); + long destination = target.getBufferPtr(); + Vect.memcpy(destination, source, HEADER_SIZE); + + byte splitFlags = Unsafe.getUnsafe().getByte(source + HEADER_OFFSET_FLAGS); + if (deferCommit) { + splitFlags |= FLAG_DEFER_COMMIT; + } else { + splitFlags &= ~FLAG_DEFER_COMMIT; + } + Unsafe.getUnsafe().putByte(destination + HEADER_OFFSET_FLAGS, splitFlags); + Unsafe.getUnsafe().putShort(destination + 6, (short) 1); + Unsafe.getUnsafe().putInt(destination + 8, messageSize - HEADER_SIZE); + + long writeAddress = destination + HEADER_SIZE; + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaStart); + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaCount); + if (deltaEntriesLength > 0) { + Vect.memcpy(writeAddress, source + deltaEntriesStart, deltaEntriesLength); + writeAddress += deltaEntriesLength; + } + Vect.memcpy(writeAddress, source + tableBodyOffset, tableBodyLength); + writeAddress += tableBodyLength; + assert writeAddress == destination + messageSize; + + target.setBufferPos(messageSize); + return messageSize; + } + public int encode(QwpTableBuffer tableBuffer) { buffer.reset(); writeHeader(1, 0); @@ -148,6 +210,17 @@ public int getDeltaEntriesStart() { return deltaEntriesStart; } + public int getSplitMessageSize(int tableBodyLength, int confirmedMaxId, int batchMaxId) { + if (tableBodyLength < 0) { + throw new IllegalArgumentException("tableBodyLength must be non-negative"); + } + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + return splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; @@ -170,4 +243,35 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { + if (splitDeltaCount == 0) { + return 0; + } + if (splitDeltaStart != deltaStart || splitDeltaCount != deltaCount) { + throw new IllegalStateException("split delta does not match the staged message" + + " [stagedStart=" + deltaStart + + ", stagedCount=" + deltaCount + + ", splitStart=" + splitDeltaStart + + ", splitCount=" + splitDeltaCount + ']'); + } + return deltaEntriesEnd - deltaEntriesStart; + } + + private int splitMessageSize( + int tableBodyLength, + int splitDeltaStart, + int splitDeltaCount, + int deltaEntriesLength + ) { + long messageSize = (long) HEADER_SIZE + + NativeBufferWriter.varintSize(splitDeltaStart) + + NativeBufferWriter.varintSize(splitDeltaCount) + + deltaEntriesLength + + tableBodyLength; + if (messageSize > Integer.MAX_VALUE) { + throw new OutOfMemoryError("split QWP message size overflow: " + messageSize); + } + return (int) messageSize; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c22f01ec..696e168e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -172,9 +172,9 @@ public class QwpWebSocketSender implements Sender { private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; // Per-table encoded body byte counts captured during flushPendingRows' combined - // encode, reused by flushPendingRowsSplit to size each split frame arithmetically - // instead of re-encoding the batch a second time. Cleared and repopulated on every - // flush; only consumed on the (exceptional) split path. Reused to stay zero-GC. + // encode. flushPendingRowsSplit uses them both for preflight sizing and to walk + // the staged body slices without encoding the batch a second time. Cleared and + // repopulated on every flush; reused to stay zero-GC. private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) @@ -3497,15 +3497,13 @@ private void flushPendingRows(boolean deferCommit) { encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, symbolDeltaBaseline(), currentBatchMaxSymbolId); - // Record each table's encoded body size (position delta across addTable) so - // the split path can size its per-table frames arithmetically rather than - // re-encoding the whole batch. Body encoding is context-free (a table's bytes - // don't depend on its siblings or the delta section), so the size a table - // takes here equals the size it takes in its own split frame. Only consumed - // when the batch overflows the cap; the capture is a couple of int ops per - // table on the common path. + // Record each table's encoded body size (position delta across addTable). + // When the batch needs splitting, these lengths delimit immutable body + // slices in the combined encoder buffer for direct frame assembly. The + // capture is a couple of int ops per table on the common path. splitFrameBodyBytes.clear(); - int bodyStart = encoder.getBuffer().getPosition(); + int combinedBodyStart = encoder.getBuffer().getPosition(); + int bodyStart = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3538,10 +3536,9 @@ private void flushPendingRows(boolean deferCommit) { // the next flush picks up the new cap. int cap = serverMaxBatchSize; if (cap > 0 && messageSize > cap) { - // The combined frame's delta-entry bytes are byte-identical to the first - // split frame's (same baseline + batch max), so capture the length now - // for the arithmetic frame-sizing in flushPendingRowsSplit. - flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen(), cap); + // Keep the completed combined frame staged in the encoder while the + // split path copies its delta entries and table-body slices. + flushPendingRowsSplit(keys, deferCommit, combinedBodyStart, cap); return; } @@ -3593,7 +3590,12 @@ private void flushPendingRows(boolean deferCommit) { * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen, int cap) { + private void flushPendingRowsSplit( + ObjList keys, + boolean deferCommit, + int combinedBodyStart, + int cap + ) { if (LOG.isDebugEnabled()) { LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); } @@ -3609,14 +3611,10 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // front makes the split all-or-nothing: either every frame fits and all // publish, or none publish and we throw with nothing stranded. // - // Each split frame's size is derived ARITHMETICALLY from the combined encode - // flushPendingRows already performed -- header + the two delta-section varints - // + the delta entries (byte-identical to the combined frame's when this frame - // carries them) + the table's own body bytes (captured in splitFrameBodyBytes, - // context-free so identical here and in its solo frame) -- rather than - // re-encoding every table a second time. simBaseline mirrors the publish loop's - // baseline advance (advanceSentMaxSymbolId), so each size equals the frame the - // publish loop will build; this pass mutates no delta/persist state. + // Each split frame's size is derived from the combined encode flushPendingRows + // already performed. simBaseline mirrors the publish loop's baseline advance + // (advanceSentMaxSymbolId), so each size equals the frame the staged-slice + // assembler will build; this pass mutates no delta/persist state. int nonEmptyCount = 0; int simBaseline = symbolDeltaBaseline(); int bodyIdx = 0; @@ -3630,13 +3628,8 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm continue; } nonEmptyCount++; - int deltaStart = simBaseline + 1; - int deltaCount = Math.max(0, currentBatchMaxSymbolId - simBaseline); - int messageSize = QwpConstants.HEADER_SIZE - + NativeBufferWriter.varintSize(deltaStart) - + NativeBufferWriter.varintSize(deltaCount) - + (deltaCount > 0 ? combinedDeltaEntriesLen : 0) - + splitFrameBodyBytes.getQuick(bodyIdx); + int messageSize = encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(bodyIdx), simBaseline, currentBatchMaxSymbolId); bodyIdx++; if (messageSize > cap) { resetTableBuffersAfterFlush(keys); @@ -3653,6 +3646,8 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm } int sent = 0; + bodyIdx = 0; + int tableBodyOffset = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3667,35 +3662,38 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm boolean isLast = (sent == nonEmptyCount); boolean deferThis = deferCommit || !isLast; - encoder.setDeferCommit(deferThis); - // Each split frame emits the delta above sentMaxSymbolId; the first - // frame ships the whole batch's new ids and advances the baseline, so - // the remaining frames carry an empty delta and just reference ids the - // first frame already registered. - encoder.beginMessage(1, globalSymbolDictionary, - symbolDeltaBaseline(), currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); - QwpBufferWriter buffer = encoder.getBuffer(); + int tableBodyLength = splitFrameBodyBytes.getQuick(bodyIdx++); + // Persist before touching activeBuffer. If the write-ahead fails, the + // caller can retry with both the source rows and active microbatch + // unchanged. The first frame carries the batch's new symbols; later + // frames are no-ops once the baseline has advanced. + persistNewSymbolsBeforePublish(); + ensureActiveBufferReady(); + // The combined encoder buffer remains immutable for the whole split. + // Assemble this frame directly into the active microbatch: patched + // header + staged delta bytes + the staged table-body slice. No row or + // column is encoded a second time. + int messageSize = encoder.copySplitMessage( + activeBuffer, + tableBodyOffset, + tableBodyLength, + deferThis, + symbolDeltaBaseline(), + currentBatchMaxSymbolId + ); + tableBodyOffset += tableBodyLength; // The pre-flight pass above already verified every split frame fits the // cap, so none can be found oversized here -- which is what keeps this // loop from publishing (and stranding) a deferred prefix before an // oversized table. Both passes size against the SAME snapshot cap, so a // mid-flush failover cannot make them disagree; the assert therefore only // catches a genuine divergence between the pre-flight arithmetic and the - // real encode (a future bug), not a cap race. It deliberately does NOT + // staged assembler (a future bug), not a cap race. It deliberately does NOT // reset+throw here, because by this point a prefix may already be on the ring. assert messageSize <= cap : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; - // Write-ahead persist before publish (see flushPendingRows). The - // first split frame carries the batch's new symbols; the rest are - // no-ops once the baseline has advanced past them. - persistNewSymbolsBeforePublish(); - ensureActiveBufferReady(); - activeBuffer.ensureCapacity(messageSize); - activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); // Frame queued: advance so the next split frame's delta starts above diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index b21d2e25..5acc0cf9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; @@ -1285,30 +1286,16 @@ public void testReset() throws Exception { } @Test - public void testTableBodyEncodingIsContextFree() throws Exception { - // The split-flush path (QwpWebSocketSender.flushPendingRowsSplit) sizes each - // per-table frame ARITHMETICALLY from splitFrameBodyBytes -- the body byte - // count captured during the COMBINED encode in flushPendingRows -- instead of - // re-encoding to measure. That is sound only while a table's body bytes are - // context-free: identical whether the table is encoded solo or as the k-th - // table after other tables and the delta section. Today every column encoder - // is stateless and symbol cells carry absolute global ids, so the property - // holds; this pins it so a future column encoder that ever carried - // cross-table state (which would break the arithmetic sizing and could strand - // a deferred prefix mid-split) fails loudly here rather than silently in - // production. + public void testSplitMessageCopiesStagedTableBodies() throws Exception { assertMemoryLeak(() -> { try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + MicrobatchBuffer target = new MicrobatchBuffer(64); QwpTableBuffer t0 = new QwpTableBuffer("alpha"); - QwpTableBuffer t1 = new QwpTableBuffer("bravo"); - QwpTableBuffer t2 = new QwpTableBuffer("charlie")) { + QwpTableBuffer t1 = new QwpTableBuffer("bravo")) { GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); int aapl = dict.getOrAddSymbol("AAPL"); // 0 int goog = dict.getOrAddSymbol("GOOG"); // 1 - int msft = dict.getOrAddSymbol("MSFT"); // 2 - // Distinct schemas + a symbol column (absolute global ids) so a - // positional / cross-table encoder bug would shift the body bytes. t0.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("AAPL", aapl); t0.getOrCreateColumn("v", TYPE_LONG, false).addLong(1L); t0.nextRow(); @@ -1318,35 +1305,56 @@ public void testTableBodyEncodingIsContextFree() throws Exception { t1.getOrCreateColumn("s", TYPE_VARCHAR, true).addString("hello"); t1.nextRow(); - t2.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("MSFT", msft); - t2.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP).addLong(1_000_000L); - t2.nextRow(); - - QwpTableBuffer[] tables = {t0, t1, t2}; - int confirmedMaxId = -1; - int batchMaxId = 2; - - // Combined encode: capture each table's body bytes exactly as - // flushPendingRows' splitFrameBodyBytes does (position delta per addTable). - int[] combinedBody = new int[tables.length]; - encoder.beginMessage(tables.length, dict, confirmedMaxId, batchMaxId); - int bodyStart = encoder.getBuffer().getPosition(); - for (int i = 0; i < tables.length; i++) { - encoder.addTable(tables[i]); - int bodyEnd = encoder.getBuffer().getPosition(); - combinedBody[i] = bodyEnd - bodyStart; - bodyStart = bodyEnd; + encoder.beginMessage(2, dict, -1, 1); + int t0BodyOffset = encoder.getBuffer().getPosition(); + encoder.addTable(t0); + int t0BodyLength = encoder.getBuffer().getPosition() - t0BodyOffset; + int t1BodyOffset = encoder.getBuffer().getPosition(); + encoder.addTable(t1); + int t1BodyLength = encoder.getBuffer().getPosition() - t1BodyOffset; + encoder.finishMessage(); + long staged = encoder.getBuffer().getBufferPtr(); + + // Erase the source tables after their one combined encode. Split + // assembly must still reproduce both bodies from staged bytes. + t0.clear(); + t1.clear(); + + int firstSize = encoder.copySplitMessage(target, t0BodyOffset, t0BodyLength, + true, -1, 1); + long first = target.getBufferPtr(); + Assert.assertEquals(firstSize, target.getBufferPos()); + Assert.assertEquals(1, Unsafe.getUnsafe().getShort(first + 6)); + Assert.assertEquals(firstSize - HEADER_SIZE, Unsafe.getUnsafe().getInt(first + 8)); + Assert.assertEquals(FLAG_DEFER_COMMIT, + (byte) (Unsafe.getUnsafe().getByte(first + HEADER_OFFSET_FLAGS) & FLAG_DEFER_COMMIT)); + Cursor cursor = new Cursor(first + HEADER_SIZE); + Assert.assertEquals(0, cursor.readVarint()); + Assert.assertEquals(2, cursor.readVarint()); + Assert.assertEquals("AAPL", cursor.readString()); + Assert.assertEquals("GOOG", cursor.readString()); + for (int i = 0; i < t0BodyLength; i++) { + Assert.assertEquals("first staged table body byte " + i, + Unsafe.getUnsafe().getByte(staged + t0BodyOffset + i), + Unsafe.getUnsafe().getByte(cursor.address + i)); } - // Solo encode each table under the SAME baseline/batch max, as the - // split publish loop does, and assert the body bytes match the capture. - for (int i = 0; i < tables.length; i++) { - encoder.beginMessage(1, dict, confirmedMaxId, batchMaxId); - int soloBodyStart = encoder.getBuffer().getPosition(); - encoder.addTable(tables[i]); - int soloBody = encoder.getBuffer().getPosition() - soloBodyStart; - Assert.assertEquals("table " + i + " body must encode identically solo vs combined " - + "(splitFrameBodyBytes relies on a context-free body)", combinedBody[i], soloBody); + target.reset(); + int secondSize = encoder.copySplitMessage(target, t1BodyOffset, t1BodyLength, + false, 1, 1); + long second = target.getBufferPtr(); + Assert.assertEquals(secondSize, target.getBufferPos()); + Assert.assertEquals(1, Unsafe.getUnsafe().getShort(second + 6)); + Assert.assertEquals(secondSize - HEADER_SIZE, Unsafe.getUnsafe().getInt(second + 8)); + Assert.assertEquals(0, + Unsafe.getUnsafe().getByte(second + HEADER_OFFSET_FLAGS) & FLAG_DEFER_COMMIT); + cursor = new Cursor(second + HEADER_SIZE); + Assert.assertEquals(2, cursor.readVarint()); + Assert.assertEquals(0, cursor.readVarint()); + for (int i = 0; i < t1BodyLength; i++) { + Assert.assertEquals("second staged table body byte " + i, + Unsafe.getUnsafe().getByte(staged + t1BodyOffset + i), + Unsafe.getUnsafe().getByte(cursor.address + i)); } } }); From f12c934e8c58134328cc357db5ec260c279d65aa Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 23:27:52 +0100 Subject: [PATCH 73/78] Clean up QWP delta dictionary tests --- .../qwp/client/DeltaDictCatchUpTest.java | 67 +- .../qwp/client/DeltaDictRecoveryTest.java | 113 +- .../cutlass/qwp/client/QwpWireTestUtils.java | 95 ++ .../sf/cursor/PersistedSymbolDictTest.java | 1004 +++++++---------- 4 files changed, 540 insertions(+), 739 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 21fdcd58..400ea1d3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -31,9 +31,6 @@ import org.junit.Test; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -329,19 +326,6 @@ private static String symbolName(int i) { return "symbol" + (1000 + i); } - private static int readVarint(byte[] buf, int[] pos) { - int result = 0; - int shift = 0; - while (pos[0] < buf.length) { - int b = buf[pos[0]++] & 0xFF; - result |= (b & 0x7F) << shift; - if ((b & 0x80) == 0) return result; - shift += 7; - if (shift > 28) throw new IllegalStateException("varint too long"); - } - throw new IllegalStateException("varint truncated"); - } - private static void waitFor(BoolCondition cond, long timeoutMillis) { long deadline = System.currentTimeMillis() + timeoutMillis; while (System.currentTimeMillis() < deadline) { @@ -412,9 +396,9 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); - accumulate(data, dict); + QwpWireTestUtils.accumulateDeltaDictionary(data, dict); if (connNumber == 2) { - if (tableCount(data) == 0) { + if (QwpWireTestUtils.tableCount(data) == 0) { sawZeroTableFrameOnConn2 = true; // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; @@ -427,7 +411,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien // frame arriving ahead of the new one -- that replay carries its // original deltaStart 0 and does not trip the flag. int[] pos = {12}; - if (readVarint(data, pos) >= 1) { + if (QwpWireTestUtils.readVarint(data, pos) >= 1) { conn2SawDeltaAboveBaseline = true; } } @@ -437,7 +421,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); // Drop the first connection right after ACKing its only frame, // forcing the sender to reconnect onto a fresh dictionary. if (connNumber == 1) { @@ -451,41 +435,6 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } } - private static void accumulate(byte[] frame, List dict) { - final byte FLAG_DELTA_SYMBOL_DICT = 0x08; - if (frame.length < 12 || (frame[5] & FLAG_DELTA_SYMBOL_DICT) == 0) { - return; - } - int[] pos = {12}; // just past the 12-byte QWP header - int deltaStart = readVarint(frame, pos); - int deltaCount = readVarint(frame, pos); - while (dict.size() < deltaStart) { - dict.add(null); // null-pad, mirroring the server - } - for (int i = 0; i < deltaCount; i++) { - int len = readVarint(frame, pos); - String symbol = new String(frame, pos[0], len, StandardCharsets.UTF_8); - pos[0] += len; - int idx = deltaStart + i; - while (dict.size() <= idx) { - dict.add(null); - } - dict.set(idx, symbol); - } - } - - private static byte[] buildAck(long seq) { - byte[] buf = new byte[1 + 8 + 2]; - ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); - bb.put((byte) 0x00); - bb.putLong(seq); - bb.putShort((short) 0); - return buf; - } - - private static int tableCount(byte[] frame) { - return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); - } } /** @@ -523,7 +472,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(CatchUpHandler.buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); if (connectionsAccepted.get() == 1) { // Connection 1 registered the big symbol. Shrink the cap so the // reconnect's catch-up budget can't fit it, then drop to force @@ -585,8 +534,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); - CatchUpHandler.accumulate(data, dict); - if (connNumber == 2 && CatchUpHandler.tableCount(data) == 0) { + QwpWireTestUtils.accumulateDeltaDictionary(data, dict); + if (connNumber == 2 && QwpWireTestUtils.tableCount(data) == 0) { zeroTableFramesOnConn2++; } try { @@ -594,7 +543,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(CatchUpHandler.buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); // Drop connection 1 only once it has learned the entire batch, so // the sender's sent-dictionary mirror is complete and the reconnect // catch-up must replay a dictionary larger than the batch cap. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index b14b0708..885a3699 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -41,10 +41,8 @@ import org.junit.Test; import java.io.IOException; -import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -1099,7 +1097,8 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti } Assert.assertEquals("two colliding symbols must persist as two entries", 2, persistedSize); - // Phase 2: recover. The seeded producer id space must match pd.size(). + // Phase 2: recover, then introduce a third symbol. The wire must extend + // the two recovered ids at id 2 rather than overwrite id 1. DictReconstructingHandler handler = new DictReconstructingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); @@ -1107,12 +1106,16 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; try (Sender s2 = Sender.fromConfig(cfg)) { - Assert.assertEquals("recovered producer dictionary must match the persisted " - + "entry count (not de-duped), else the delta baseline desyncs from the mirror", - persistedSize, globalDictSize(s2)); - Assert.assertEquals("delta baseline must resume at the persisted tip", - persistedSize - 1, intField(s2, "sentMaxSymbolId")); + s2.table("m").symbol("s", "tail").longColumn("v", 3L).atNow(); + long fsn = s2.flushAndGetSequence(); + Assert.assertTrue("the post-recovery frame must be acknowledged", + s2.awaitAckedFsn(fsn, 5_000)); } + Assert.assertEquals("the new symbol delta must continue after both recovered entries", + persistedSize, handler.lastDataDeltaStart()); + Assert.assertEquals("the reconstructed dictionary must retain both colliding entries " + + "and append the new symbol at id 2", + Arrays.asList("?", "?", "tail"), handler.dictSnapshot()); } }); } @@ -1369,19 +1372,6 @@ public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Ex }); } - private static int globalDictSize(Sender sender) throws Exception { - Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); - f.setAccessible(true); - Object dict = f.get(sender); - return (int) dict.getClass().getMethod("size").invoke(dict); - } - - private static int intField(Sender sender, String name) throws Exception { - Field f = sender.getClass().getDeclaredField(name); - f.setAccessible(true); - return f.getInt(sender); - } - private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); @@ -1391,23 +1381,6 @@ private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws java.nio.file.Files.write(path, buf); } - private static int readVarint(byte[] buf, int[] pos) { - int result = 0; - int shift = 0; - while (pos[0] < buf.length) { - int b = buf[pos[0]++] & 0xFF; - result |= (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return result; - } - shift += 7; - if (shift > 28) { - throw new IllegalStateException("varint too long"); - } - } - throw new IllegalStateException("varint truncated"); - } - /** * Reconstructs the per-connection symbol dictionary from delta sections, * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding. @@ -1418,6 +1391,7 @@ private static class DictReconstructingHandler implements TestWebSocketServer.We private final List dict = new ArrayList<>(); private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; + private volatile int lastDataDeltaStart = -1; synchronized List ackSequenceStarts() { return new ArrayList<>(ackSequenceStarts); @@ -1431,6 +1405,10 @@ synchronized int maxDictSize() { return dict.size(); } + int lastDataDeltaStart() { + return lastDataDeltaStart; + } + @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { boolean newConnection = currentClient != client; @@ -1439,59 +1417,23 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien dict.clear(); // fresh server dictionary per connection nextSeq.set(0); } - accumulate(data); - if (tableCount(data) == 0 && hasDelta(data)) { + QwpWireTestUtils.accumulateDeltaDictionary(data, dict); + int tableCount = QwpWireTestUtils.tableCount(data); + if (tableCount == 0 && QwpWireTestUtils.hasDelta(data)) { sawCatchUpFrame = true; + } else if (tableCount > 0 && QwpWireTestUtils.hasDelta(data)) { + lastDataDeltaStart = QwpWireTestUtils.readVarint(data, new int[]{12}); } try { long ackSequence = nextSeq.getAndIncrement(); if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); } catch (IOException e) { throw new RuntimeException(e); } } - - private static byte[] buildAck(long seq) { - byte[] buf = new byte[1 + 8 + 2]; - ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); - bb.put((byte) 0x00); - bb.putLong(seq); - bb.putShort((short) 0); - return buf; - } - - private static boolean hasDelta(byte[] frame) { - return frame.length >= 12 && (frame[5] & 0x08) != 0; - } - - private static int tableCount(byte[] frame) { - return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); - } - - private void accumulate(byte[] frame) { - if (!hasDelta(frame)) { - return; - } - int[] pos = {12}; - int deltaStart = readVarint(frame, pos); - int deltaCount = readVarint(frame, pos); - while (dict.size() < deltaStart) { - dict.add(null); - } - for (int i = 0; i < deltaCount; i++) { - int len = readVarint(frame, pos); - String sym = new String(frame, pos[0], len, StandardCharsets.UTF_8); - pos[0] += len; - int idx = deltaStart + i; - while (dict.size() <= idx) { - dict.add(null); - } - dict.set(idx, sym); - } - } } private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { @@ -1510,20 +1452,11 @@ private static class CountingHandler implements TestWebSocketServer.WebSocketSer public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { frames.incrementAndGet(); try { - client.sendBinary(buildAck(nextSeq.getAndIncrement())); + client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement())); } catch (IOException e) { throw new RuntimeException(e); } } - - private static byte[] buildAck(long seq) { - byte[] buf = new byte[1 + 8 + 2]; - ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); - bb.put((byte) 0x00); - bb.putLong(seq); - bb.putShort((short) 0); - return buf; - } } /** * Short-writes the FIRST dictionary append after it is armed, modelling a disk that diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java new file mode 100644 index 00000000..2d2cc0b1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java @@ -0,0 +1,95 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE; + +final class QwpWireTestUtils { + + static void accumulateDeltaDictionary(byte[] frame, List dictionary) { + if (!hasDelta(frame)) { + return; + } + int[] position = {HEADER_SIZE}; + int deltaStart = readVarint(frame, position); + int deltaCount = readVarint(frame, position); + while (dictionary.size() < deltaStart) { + dictionary.add(null); + } + for (int i = 0; i < deltaCount; i++) { + int length = readVarint(frame, position); + String symbol = new String(frame, position[0], length, StandardCharsets.UTF_8); + position[0] += length; + int symbolId = deltaStart + i; + while (dictionary.size() <= symbolId) { + dictionary.add(null); + } + dictionary.set(symbolId, symbol); + } + } + + static byte[] buildAck(long sequence) { + byte[] buffer = new byte[1 + Long.BYTES + Short.BYTES]; + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN); + byteBuffer.put((byte) 0x00); + byteBuffer.putLong(sequence); + byteBuffer.putShort((short) 0); + return buffer; + } + + static boolean hasDelta(byte[] frame) { + return frame.length >= HEADER_SIZE && (frame[5] & FLAG_DELTA_SYMBOL_DICT) != 0; + } + + static int readVarint(byte[] buffer, int[] position) { + int result = 0; + int shift = 0; + while (position[0] < buffer.length) { + int b = buffer[position[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + if (shift > 28) { + throw new IllegalStateException("varint too long"); + } + } + throw new IllegalStateException("varint truncated"); + } + + static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + + private QwpWireTestUtils() { + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 280155a7..aea6d1af 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -32,8 +32,11 @@ import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Assume; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -48,51 +51,41 @@ public class PersistedSymbolDictTest { private static final int CRC_SIZE = 4; private static final int HEADER_SIZE = 8; + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + @Test public void testAppendPersistsAcrossReopen() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(d); - try { - Assert.assertEquals(0, d.size()); - d.appendSymbol("AAPL"); - d.appendSymbol("GOOG"); - d.appendSymbol("MSFT"); - Assert.assertEquals(3, d.size()); - } finally { - d.close(); - } + Assert.assertEquals(0, d.size()); + d.appendSymbol("AAPL"); + d.appendSymbol("GOOG"); + d.appendSymbol("MSFT"); + Assert.assertEquals(3, d.size()); + } - // Reopen: entries recovered in id order. - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + // Reopen: entries recovered in id order. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(reopened); - try { - Assert.assertEquals(3, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals(3, symbols.size()); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("GOOG", symbols.getQuick(1)); - Assert.assertEquals("MSFT", symbols.getQuick(2)); - Assert.assertTrue(reopened.loadedEntriesLen() > 0); - - // Appending after recovery continues from the recovered tip. - reopened.appendSymbol("TSLA"); - Assert.assertEquals(4, reopened.size()); - } finally { - reopened.close(); - } + Assert.assertEquals(3, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(3, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertTrue(reopened.loadedEntriesLen() > 0); + + // Appending after recovery continues from the recovered tip. + reopened.appendSymbol("TSLA"); + Assert.assertEquals(4, reopened.size()); + } - PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(4, third.size()); - Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3)); - } finally { - third.close(); - } - } finally { - rmDir(dir); + try (PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(4, third.size()); + Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3)); } }); } @@ -107,48 +100,33 @@ public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { // appendRawEntries into a fresh dict, and assert the recovered symbols // match -- including an empty entry mid-range. assertMemoryLeak(() -> { - Path src = Files.createTempDirectory("qwp-symdict-src"); - Path dst = Files.createTempDirectory("qwp-symdict-dst"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range - dict.getOrAddSymbol("MSFT"); // id 2 - - PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + Path src = newFolder("qwp-symdict-src"); + Path dst = newFolder("qwp-symdict-dst"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + + try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) { encoded.appendSymbols(dict, 0, 2); - encoded.close(); + } - // Reopen to obtain the on-disk entry region [len][utf8]... verbatim, - // then replay it byte-for-byte into a fresh dict via appendRawEntries. - PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); - try { - PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); - try { - raw.appendRawEntries(reopened.loadedEntriesAddr(), - reopened.loadedEntriesLen(), reopened.size()); - Assert.assertEquals(3, raw.size()); - } finally { - raw.close(); - } - } finally { - reopened.close(); - } + // Reopen to obtain the on-disk entry region [len][utf8]... verbatim, + // then replay it byte-for-byte into a fresh dict via appendRawEntries. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString())) { + raw.appendRawEntries(reopened.loadedEntriesAddr(), + reopened.loadedEntriesLen(), reopened.size()); + Assert.assertEquals(3, raw.size()); + } - // The raw-appended dict must recover the same dense symbols. - PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString()); - try { - Assert.assertEquals(3, recovered.size()); - ObjList symbols = recovered.readLoadedSymbols(); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("", symbols.getQuick(1)); - Assert.assertEquals("MSFT", symbols.getQuick(2)); - } finally { - recovered.close(); - } - } finally { - rmDir(src); - rmDir(dst); + // The raw-appended dict must recover the same dense symbols. + try (PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString())) { + Assert.assertEquals(3, recovered.size()); + ObjList symbols = recovered.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); } }); } @@ -166,39 +144,29 @@ public void testAppendRawEntriesRejectsInconsistentTripleUnderAssertions() throw assert assertionsEnabled = true; Assume.assumeTrue("the guard is assert-gated; only observable under -ea", assertionsEnabled); assertMemoryLeak(() -> { - Path src = Files.createTempDirectory("qwp-symdict-src"); - Path dst = Files.createTempDirectory("qwp-symdict-dst"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol("MSFT"); // id 1 - dict.getOrAddSymbol("GOOG"); // id 2 - PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + Path src = newFolder("qwp-symdict-src"); + Path dst = newFolder("qwp-symdict-dst"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + dict.getOrAddSymbol("GOOG"); // id 2 + try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) { encoded.appendSymbols(dict, 0, 2); - encoded.close(); + } - // Grab the on-disk entry region (3 entries) verbatim, then feed it back - // with a count of 1: validateRawEntries stops one entry in with - // src < srcLimit and throws "under-filled", before any bytes are written. - PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + // Grab the on-disk entry region (3 entries) verbatim, then feed it back + // with a count of 1: validateRawEntries stops one entry in with + // src < srcLimit and throws "under-filled", before any bytes are written. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString())) { try { - PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); - try { - raw.appendRawEntries(reopened.loadedEntriesAddr(), reopened.loadedEntriesLen(), 1); - Assert.fail("an inconsistent (len,count) triple must be rejected"); - } catch (IllegalStateException expected) { - Assert.assertTrue("message names the under-filled buffer: " + expected.getMessage(), - expected.getMessage().contains("under-filled")); - Assert.assertEquals("a rejected triple must write nothing", 0, raw.size()); - } finally { - raw.close(); - } - } finally { - reopened.close(); + raw.appendRawEntries(reopened.loadedEntriesAddr(), reopened.loadedEntriesLen(), 1); + Assert.fail("an inconsistent (len,count) triple must be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue("message names the under-filled buffer: " + expected.getMessage(), + expected.getMessage().contains("under-filled")); + Assert.assertEquals("a rejected triple must write nothing", 0, raw.size()); } - } finally { - rmDir(src); - rmDir(dst); } }); } @@ -214,63 +182,49 @@ public void testAppendRawEntriesShortWriteThrowsWithoutAdvancingIsIdempotentOnRe // the persist-FAILURE trigger, whose idempotency the write-ahead // (persistNewSymbolsBeforePublish, resuming from pd.size()) relies on. assertMemoryLeak(() -> { - Path src = Files.createTempDirectory("qwp-symdict-src"); - Path dst = Files.createTempDirectory("qwp-symdict-dst"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol("MSFT"); // id 1 - - // Encode the range once to obtain its on-disk [len][utf8]... bytes. - PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + Path src = newFolder("qwp-symdict-src"); + Path dst = newFolder("qwp-symdict-dst"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + + // Encode the range once to obtain its on-disk [len][utf8]... bytes. + try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) { encoded.appendSymbols(dict, 0, 1); - encoded.close(); + } - PersistedSymbolDict source = PersistedSymbolDict.open(src.toString()); - try { - long addr = source.loadedEntriesAddr(); - int rawLen = source.loadedEntriesLen(); - int count = source.size(); + try (PersistedSymbolDict source = PersistedSymbolDict.open(src.toString())) { + long addr = source.loadedEntriesAddr(); + int rawLen = source.loadedEntriesLen(); + int count = source.size(); - ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); - PersistedSymbolDict d = PersistedSymbolDict.open(ff, dst.toString()); + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + try (PersistedSymbolDict d = PersistedSymbolDict.open(ff, dst.toString())) { Assert.assertNotNull(d); + ff.armed = true; // the next entry append lands short try { - ff.armed = true; // the next entry append lands short - try { - d.appendRawEntries(addr, rawLen, count); - Assert.fail("a short write must throw"); - } catch (IllegalStateException expected) { - Assert.assertTrue("short-write error: " + expected.getMessage(), - expected.getMessage().contains("short write")); - } - // The throw preceded the size/offset advance: nothing persisted. - Assert.assertEquals("short write must NOT advance size", 0, d.size()); - - // Retry the SAME bytes (the facade auto-disarmed): the write - // lands at the unchanged offset, overwriting the torn prefix. d.appendRawEntries(addr, rawLen, count); - Assert.assertEquals(2, d.size()); - } finally { - d.close(); + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); } - } finally { - source.close(); - } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); - // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). - PersistedSymbolDict reopened = PersistedSymbolDict.open(dst.toString()); - try { - Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("MSFT", symbols.getQuick(1)); - } finally { - reopened.close(); + // Retry the SAME bytes (the facade auto-disarmed): the write + // lands at the unchanged offset, overwriting the torn prefix. + d.appendRawEntries(addr, rawLen, count); + Assert.assertEquals(2, d.size()); } - } finally { - rmDir(src); - rmDir(dst); + } + + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dst.toString())) { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("MSFT", symbols.getQuick(1)); } }); } @@ -283,52 +237,39 @@ public void testAppendSymbolsBatchWritesDenseRange() throws Exception { // batched call keyed off size() must continue densely (the resume-from- // durable-size contract the producer relies on). assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range - dict.getOrAddSymbol("MSFT"); // id 2 - dict.getOrAddSymbol("TSLA"); // id 3 - - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbols(dict, 0, 3); // one write for all four ids - Assert.assertEquals(4, d.size()); - d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op - Assert.assertEquals(4, d.size()); - } finally { - d.close(); - } + Path dir = newFolder("qwp-symdict"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + dict.getOrAddSymbol("TSLA"); // id 3 + + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbols(dict, 0, 3); // one write for all four ids + Assert.assertEquals(4, d.size()); + d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op + Assert.assertEquals(4, d.size()); + } - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(4, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals(4, symbols.size()); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("", symbols.getQuick(1)); - Assert.assertEquals("MSFT", symbols.getQuick(2)); - Assert.assertEquals("TSLA", symbols.getQuick(3)); - - // A follow-on batch keyed off the recovered size continues - // the dense sequence without a gap or duplicate. - dict.getOrAddSymbol("NVDA"); // id 4 - reopened.appendSymbols(dict, reopened.size(), 4); - Assert.assertEquals(5, reopened.size()); - } finally { - reopened.close(); - } + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(4, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(4, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertEquals("TSLA", symbols.getQuick(3)); + + // A follow-on batch keyed off the recovered size continues + // the dense sequence without a gap or duplicate. + dict.getOrAddSymbol("NVDA"); // id 4 + reopened.appendSymbols(dict, reopened.size(), 4); + Assert.assertEquals(5, reopened.size()); + } - PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(5, third.size()); - Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4)); - } finally { - third.close(); - } - } finally { - rmDir(dir); + try (PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(5, third.size()); + Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4)); } }); } @@ -342,47 +283,37 @@ public void testAppendSymbolsShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry // gap-free, duplicate-free dictionary. A regression that advanced size before // the written==len check would strand a torn/duplicated dictionary here. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol("GOOG"); // id 1 + Path dir = newFolder("qwp-symdict"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("GOOG"); // id 1 - ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); - PersistedSymbolDict d = PersistedSymbolDict.open(ff, dir.toString()); + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + try (PersistedSymbolDict d = PersistedSymbolDict.open(ff, dir.toString())) { Assert.assertNotNull(d); + ff.armed = true; // the next entry append lands short try { - ff.armed = true; // the next entry append lands short - try { - d.appendSymbols(dict, 0, 1); - Assert.fail("a short write must throw"); - } catch (IllegalStateException expected) { - Assert.assertTrue("short-write error: " + expected.getMessage(), - expected.getMessage().contains("short write")); - } - // The throw preceded the size/offset advance: nothing persisted. - Assert.assertEquals("short write must NOT advance size", 0, d.size()); - - // Retry the SAME range (the facade auto-disarmed): re-writes at - // the unchanged offset, overwriting the torn bytes. d.appendSymbols(dict, 0, 1); - Assert.assertEquals(2, d.size()); - } finally { - d.close(); - } + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); + } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); + + // Retry the SAME range (the facade auto-disarmed): re-writes at + // the unchanged offset, overwriting the torn bytes. + d.appendSymbols(dict, 0, 1); + Assert.assertEquals(2, d.size()); + } - // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("GOOG", symbols.getQuick(1)); - } finally { - reopened.close(); - } - } finally { - rmDir(dir); + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); } }); } @@ -404,44 +335,34 @@ public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { // symbols cost N * ([len varint] + 1 utf8 byte) of entries, wrapped in one chunk. // Per-entry CRCs would add 4 * N bytes instead of 4. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - final int n = 26; // 'a'..'z' -- distinct, and one UTF-8 byte each - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - for (int i = 0; i < n; i++) { - dict.getOrAddSymbol(String.valueOf((char) ('a' + i))); - } - Assert.assertEquals(n, dict.size()); - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + final int n = 26; // 'a'..'z' -- distinct, and one UTF-8 byte each + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + for (int i = 0; i < n; i++) { + dict.getOrAddSymbol(String.valueOf((char) ('a' + i))); + } + Assert.assertEquals(n, dict.size()); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(d); - try { - d.appendSymbols(dict, 0, n - 1); - Assert.assertEquals(n, d.size()); - } finally { - d.close(); - } + d.appendSymbols(dict, 0, n - 1); + Assert.assertEquals(n, d.size()); + } - int entryBytes = n * 2; // [len=1 varint][1 utf8 byte] - int chunkHeader = 1 + varintSize(entryBytes); // entryCount=50 fits one byte - long expected = HEADER_SIZE + chunkHeader + entryBytes + CRC_SIZE; - Assert.assertEquals( - "a batch append must cost ONE chunk checksum, not one per symbol", - expected, Files.size(dir.resolve(".symbol-dict"))); + int entryBytes = n * 2; // [len=1 varint][1 utf8 byte] + int chunkHeader = 1 + varintSize(entryBytes); // entryCount=50 fits one byte + long expected = HEADER_SIZE + chunkHeader + entryBytes + CRC_SIZE; + Assert.assertEquals( + "a batch append must cost ONE chunk checksum, not one per symbol", + expected, Files.size(dir.resolve(".symbol-dict"))); - // And it must still read back symbol-for-symbol. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + // And it must still read back symbol-for-symbol. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals(n, re.size()); - ObjList got = re.readLoadedSymbols(); - for (int i = 0; i < n; i++) { - Assert.assertEquals(dict.getSymbol(i), got.getQuick(i)); - } - } finally { - re.close(); + Assert.assertEquals(n, re.size()); + ObjList got = re.readLoadedSymbols(); + for (int i = 0; i < n; i++) { + Assert.assertEquals(dict.getSymbol(i), got.getQuick(i)); } - } finally { - rmDir(dir); } }); } @@ -449,33 +370,23 @@ public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { @Test public void testMappedAppendAmortizesFlushesWithoutPositionedWrites() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(d); - try { - for (int i = 0; i < 10_000; i++) { - d.appendSymbol("sym-" + i); - } - Assert.assertEquals(10_000, d.size()); - Assert.assertEquals("production appends must write directly into the mmap", - 0L, d.appendWriteCount()); - Assert.assertEquals("ten thousand flushes must require only three mapped windows", - 3, d.appendMapGrowthCount()); - } finally { - d.close(); - } + for (int i = 0; i < 10_000; i++) { + d.appendSymbol("sym-" + i); + } + Assert.assertEquals(10_000, d.size()); + Assert.assertEquals("production appends must write directly into the mmap", + 0L, d.appendWriteCount()); + Assert.assertEquals("ten thousand flushes must require only three mapped windows", + 3, d.appendMapGrowthCount()); + } - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals(10_000, re.size()); - Assert.assertEquals("sym-9999", re.readLoadedSymbols().getQuick(9_999)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals(10_000, re.size()); + Assert.assertEquals("sym-9999", re.readLoadedSymbols().getQuick(9_999)); } }); } @@ -495,18 +406,14 @@ public void testBadMagicDegradesWithoutDestroyingTheFile() throws Exception { // fall through to openFresh(), which is O_TRUNC: a single unreadable byte // destroyed the only copy of load-bearing state. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - Path f = dir.resolve(".symbol-dict"); - byte[] original = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - Files.write(f, original); - Assert.assertNull("an unparseable existing dict must degrade to null", - PersistedSymbolDict.open(dir.toString())); - Assert.assertArrayEquals("open() must NOT destroy an existing dictionary", - original, Files.readAllBytes(f)); - } finally { - rmDir(dir); - } + Path dir = newFolder("qwp-symdict"); + Path f = dir.resolve(".symbol-dict"); + byte[] original = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + Files.write(f, original); + Assert.assertNull("an unparseable existing dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy an existing dictionary", + original, Files.readAllBytes(f)); }); } @@ -524,25 +431,21 @@ public void testBadVersionDegradesWithoutDestroyingTheFile() throws Exception { // unreplayable. Degrading to null instead leaves the bytes for the client that // does understand them. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - Path f = dir.resolve(".symbol-dict"); - PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + Path f = dir.resolve(".symbol-dict"); + try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(seed); seed.appendSymbol("a"); - seed.close(); - // Corrupt ONLY the version byte (offset 4) to an unknown value. - byte[] bytes = Files.readAllBytes(f); - bytes[4] = (byte) 99; - Files.write(f, bytes); - - Assert.assertNull("an unknown-version dict must degrade to null", - PersistedSymbolDict.open(dir.toString())); - Assert.assertArrayEquals("open() must NOT destroy a future-version dictionary", - bytes, Files.readAllBytes(f)); - } finally { - rmDir(dir); } + // Corrupt ONLY the version byte (offset 4) to an unknown value. + byte[] bytes = Files.readAllBytes(f); + bytes[4] = (byte) 99; + Files.write(f, bytes); + + Assert.assertNull("an unknown-version dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy a future-version dictionary", + bytes, Files.readAllBytes(f)); }); } @@ -553,48 +456,35 @@ public void testCloseNullsLoadedEntries() throws Exception { // dereference freed native memory. Pre-fix the pointer survived close() // non-zero. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("AAPL"); - d.close(); - - // Reopen so recovery loads the entries into native memory. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - Assert.assertTrue("recovery must load entries into native memory", - re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0); - re.close(); - Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr()); - Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen()); - } finally { - rmDir(dir); } + + // Reopen so recovery loads the entries into native memory. This test + // closes explicitly because the post-close state is the assertion target. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertTrue("recovery must load entries into native memory", + re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0); + re.close(); + Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr()); + Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen()); }); } @Test public void testEmptySymbolRoundTrips() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbol(""); - d.appendSymbol("nonempty"); - } finally { - d.close(); - } - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("", s.getQuick(0)); - Assert.assertEquals("nonempty", s.getQuick(1)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbol(""); + d.appendSymbol("nonempty"); + } + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("", s.getQuick(0)); + Assert.assertEquals("nonempty", s.getQuick(1)); } }); } @@ -610,49 +500,39 @@ public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exce // only the intact prefix (fail-clean: the send loop's torn-dict guard then // forces a resend of the rest). assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbol("s0"); - d.appendSymbol("s1"); - d.appendSymbol("s2"); - d.appendSymbol("s3"); - d.appendSymbol("s4"); - Assert.assertEquals(5, d.size()); - } finally { - d.close(); - } + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbol("s0"); + d.appendSymbol("s1"); + d.appendSymbol("s2"); + d.appendSymbol("s3"); + d.appendSymbol("s4"); + Assert.assertEquals(5, d.size()); + } - // Corrupt one byte inside the 3rd chunk's entry region (id 2). Each - // appendSymbol writes one single-entry chunk - // ([entryCount][entryBytes][len][utf8][crc32c]), and all five symbols - // are the same width, so the five chunks are equal-sized -- derive the - // stride from the file rather than hard-coding it. The byte flipped is - // the last one before the chunk's trailing CRC, so that CRC goes stale. - Path f = dir.resolve(".symbol-dict"); - byte[] bytes = Files.readAllBytes(f); - int chunkLen = (bytes.length - HEADER_SIZE) / 5; - int chunk2 = HEADER_SIZE + 2 * chunkLen; - bytes[chunk2 + chunkLen - CRC_SIZE - 1] ^= 0x7F; - Files.write(f, bytes); - - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + // Corrupt one byte inside the 3rd chunk's entry region (id 2). Each + // appendSymbol writes one single-entry chunk + // ([entryCount][entryBytes][len][utf8][crc32c]), and all five symbols + // are the same width, so the five chunks are equal-sized -- derive the + // stride from the file rather than hard-coding it. The byte flipped is + // the last one before the chunk's trailing CRC, so that CRC goes stale. + Path f = dir.resolve(".symbol-dict"); + byte[] bytes = Files.readAllBytes(f); + int chunkLen = (bytes.length - HEADER_SIZE) / 5; + int chunk2 = HEADER_SIZE + 2 * chunkLen; + bytes[chunk2 + chunkLen - CRC_SIZE - 1] ^= 0x7F; + Files.write(f, bytes); + + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - // Only the intact prefix [s0, s1] is trusted; the corrupt chunk - // and everything after it are dropped. No recovered symbol is - // the corrupted string -- the tear is DETECTED, never silently - // misattributed. - Assert.assertEquals("parse must stop at the corrupt interior chunk", 2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("s0", s.getQuick(0)); - Assert.assertEquals("s1", s.getQuick(1)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + // Only the intact prefix [s0, s1] is trusted; the corrupt chunk + // and everything after it are dropped. No recovered symbol is + // the corrupted string -- the tear is DETECTED, never silently + // misattributed. + Assert.assertEquals("parse must stop at the corrupt interior chunk", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("s0", s.getQuick(0)); + Assert.assertEquals("s1", s.getQuick(1)); } }); } @@ -668,35 +548,25 @@ public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { // forward's process-crash durability for large symbols. The file length is // now the only bound, so the write and read paths agree. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - // Just over the old 1 << 20 (1 MB) ceiling. - String big = TestUtils.repeat("x", (1 << 20) + 17); - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbol("before"); - d.appendSymbol(big); - d.appendSymbol("after"); - Assert.assertEquals(3, d.size()); - } finally { - d.close(); - } + Path dir = newFolder("qwp-symdict"); + // Just over the old 1 << 20 (1 MB) ceiling. + String big = TestUtils.repeat("x", (1 << 20) + 17); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbol("before"); + d.appendSymbol(big); + d.appendSymbol("after"); + Assert.assertEquals(3, d.size()); + } - // Recovery must load ALL three; pre-fix the reopen truncated at the - // big entry and came back with size 1 (only "before" survived). - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals("large entry must survive recovery, not be truncated", - 3, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("before", s.getQuick(0)); - Assert.assertEquals(big, s.getQuick(1)); - Assert.assertEquals("after", s.getQuick(2)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + // Recovery must load ALL three; pre-fix the reopen truncated at the + // big entry and came back with size 1 (only "before" survived). + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals("large entry must survive recovery, not be truncated", + 3, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("before", s.getQuick(0)); + Assert.assertEquals(big, s.getQuick(1)); + Assert.assertEquals("after", s.getQuick(2)); } }); } @@ -709,43 +579,30 @@ public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { // fresh-start producer is not seeded from it -- shifts the dense id->symbol // mapping and misattributes symbols on the next reconnect. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict-clean"); - try { - PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict-clean"); + try (PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(stale); - try { - stale.appendSymbol("staleX"); - stale.appendSymbol("staleY"); - Assert.assertEquals(2, stale.size()); - } finally { - stale.close(); - } + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + Assert.assertEquals(2, stale.size()); + } - // Fresh start: openClean yields an EMPTY dictionary regardless of - // the survivor, and appends from id 0 again. - PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString()); + // Fresh start: openClean yields an EMPTY dictionary regardless of + // the survivor, and appends from id 0 again. + try (PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString())) { Assert.assertNotNull(fresh); - try { - Assert.assertEquals(0, fresh.size()); - Assert.assertEquals(0, fresh.readLoadedSymbols().size()); - fresh.appendSymbol("freshA"); - Assert.assertEquals(1, fresh.size()); - } finally { - fresh.close(); - } + Assert.assertEquals(0, fresh.size()); + Assert.assertEquals(0, fresh.readLoadedSymbols().size()); + fresh.appendSymbol("freshA"); + Assert.assertEquals(1, fresh.size()); + } - // The survivor's bytes are physically gone, not just hidden: a - // subsequent recovery open() sees only the post-clean content. - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + // The survivor's bytes are physically gone, not just hidden: a + // subsequent recovery open() sees only the post-clean content. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(reopened); - try { - Assert.assertEquals(1, reopened.size()); - Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0)); - } finally { - reopened.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals(1, reopened.size()); + Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0)); } }); } @@ -753,18 +610,14 @@ public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("A"); - d.close(); - Path f = dir.resolve(".symbol-dict"); - Assert.assertTrue(Files.exists(f)); - PersistedSymbolDict.removeOrphan(dir.toString()); - Assert.assertFalse(Files.exists(f)); - } finally { - rmDir(dir); } + Path f = dir.resolve(".symbol-dict"); + Assert.assertTrue(Files.exists(f)); + PersistedSymbolDict.removeOrphan(dir.toString()); + Assert.assertFalse(Files.exists(f)); }); } @@ -785,76 +638,62 @@ public void testTooLargeToReopenDegradesWithoutOpeningTheFile() throws Exception // null) but that the file is never even OPENED: with the guard, open() returns // before openRW; without it, openExisting opens the file and parses garbage. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(seed); for (int i = 0; i < 40; i++) { seed.appendSymbol("sym" + i); } - seed.close(); - Path f = dir.resolve(".symbol-dict"); - byte[] before = Files.readAllBytes(f); - - HugeLengthFacade ff = new HugeLengthFacade(); - Assert.assertNull("a >=2GB dictionary must degrade to null", - PersistedSymbolDict.open(ff, dir.toString())); - Assert.assertEquals("the guard must short-circuit BEFORE opening the file", - 0, ff.openRwCalls); - Assert.assertArrayEquals("open() must NOT destroy or trim an oversized dictionary", - before, Files.readAllBytes(f)); - } finally { - rmDir(dir); } + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + + HugeLengthFacade ff = new HugeLengthFacade(); + Assert.assertNull("a >=2GB dictionary must degrade to null", + PersistedSymbolDict.open(ff, dir.toString())); + Assert.assertEquals("the guard must short-circuit BEFORE opening the file", + 0, ff.openRwCalls); + Assert.assertArrayEquals("open() must NOT destroy or trim an oversized dictionary", + before, Files.readAllBytes(f)); }); } @Test public void testTornTrailingEntrySelfHeals() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - // Write two complete entries, then a torn trailing record: a - // length prefix of 5 followed by only 2 bytes (crash mid-append). - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + // Write two complete entries, then a torn trailing record: a + // length prefix of 5 followed by only 2 bytes (crash mid-append). + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("one"); d.appendSymbol("two"); - d.close(); - - Path f = dir.resolve(".symbol-dict"); - long cleanLen = Files.size(f); // header + "one" + "two", no tail - Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, - StandardOpenOption.APPEND); - Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + } - // Reopen: the torn tail is ignored; only the two complete entries load. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - try { - // open() physically truncates the torn tail: the file returns - // to its clean length, so a later SHORTER append can never - // leave residue past its end that a future recovery mis-parses - // as a ghost symbol. - Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f)); - Assert.assertEquals(2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("one", s.getQuick(0)); - Assert.assertEquals("two", s.getQuick(1)); - // The next append continues from the truncated tail cleanly. - re.appendSymbol("three"); - Assert.assertEquals(3, re.size()); - } finally { - re.close(); - } + Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, + StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + + // Reopen: the torn tail is ignored; only the two complete entries load. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { + // open() physically truncates the torn tail: the file returns + // to its clean length, so a later SHORTER append can never + // leave residue past its end that a future recovery mis-parses + // as a ghost symbol. + Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f)); + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + // The next append continues from the truncated tail cleanly. + re.appendSymbol("three"); + Assert.assertEquals(3, re.size()); + } - PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(3, re2.size()); - Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2)); - } finally { - re2.close(); - } - } finally { - rmDir(dir); + try (PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(3, re2.size()); + Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2)); } }); } @@ -871,44 +710,37 @@ public void testTransientLengthFaultDegradesWithoutDestroyingTheFile() throws Ex // null (fall back to full self-sufficient frames) and leave every byte on disk // for a later attempt, once the transient clears. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("one"); d.appendSymbol("two"); - d.close(); - - Path f = dir.resolve(".symbol-dict"); - byte[] before = Files.readAllBytes(f); - Assert.assertTrue("a populated dict must exceed the header", before.length > HEADER_SIZE); - - // Reopen through a facade whose length() reports the -1 stat-error - // sentinel for the (present) file -- the branch under test. - PersistedSymbolDict reopened = PersistedSymbolDict.open(new StatFailsLengthFacade(), dir.toString()); - if (reopened != null) { - // Pre-fix: open() fell through to openFresh() and handed back a fresh - // empty dict over the now-truncated file. Close its fd so only the - // assertion below reports the failure, not a leaked descriptor. - reopened.close(); - } - Assert.assertNull("a populated dict whose length cannot be stat'd must degrade to null, not truncate", - reopened); - Assert.assertArrayEquals("a transient length-stat fault must NOT destroy the dictionary", - before, Files.readAllBytes(f)); + } + + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + Assert.assertTrue("a populated dict must exceed the header", before.length > HEADER_SIZE); + + // Reopen through a facade whose length() reports the -1 stat-error + // sentinel for the (present) file -- the branch under test. + PersistedSymbolDict reopened = PersistedSymbolDict.open(new StatFailsLengthFacade(), dir.toString()); + if (reopened != null) { + // Pre-fix: open() fell through to openFresh() and handed back a fresh + // empty dict over the now-truncated file. Close its fd so only the + // assertion below reports the failure, not a leaked descriptor. + reopened.close(); + } + Assert.assertNull("a populated dict whose length cannot be stat'd must degrade to null, not truncate", + reopened); + Assert.assertArrayEquals("a transient length-stat fault must NOT destroy the dictionary", + before, Files.readAllBytes(f)); - // Once the filesystem recovers, the SAME file reopens its intact content. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + // Once the filesystem recovers, the SAME file reopens its intact content. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals("the intact content survives the transient", 2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("one", s.getQuick(0)); - Assert.assertEquals("two", s.getQuick(1)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals("the intact content survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); } }); } @@ -928,52 +760,44 @@ public void testTruncateFailureDegradesWithoutDestroyingTheFile() throws Excepti // makes every surviving delta frame permanently unreplayable. Degrading leaves // the bytes for the next attempt, once the mount is writable again. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("one"); d.appendSymbol("two"); - d.close(); - - // Append a torn trailing record so the reopen parses [one, two], then - // finds validLen < len and tries to truncate the tail -- the branch - // under test. - Path f = dir.resolve(".symbol-dict"); - long cleanLen = Files.size(f); // header + "one" + "two", no tail - Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); - Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); - byte[] before = Files.readAllBytes(f); - - // Reopen through a facade whose truncate() fails. - Assert.assertNull("a dict whose torn tail cannot be trimmed must degrade to null", - PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString())); - Assert.assertArrayEquals("a failed truncate must NOT destroy the dictionary", - before, Files.readAllBytes(f)); - - // And once the filesystem is writable again, the SAME file recovers its - // intact prefix in full -- which is the whole point of not destroying it. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + } + + // Append a torn trailing record so the reopen parses [one, two], then + // finds validLen < len and tries to truncate the tail -- the branch + // under test. + Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + byte[] before = Files.readAllBytes(f); + + // Reopen through a facade whose truncate() fails. + Assert.assertNull("a dict whose torn tail cannot be trimmed must degrade to null", + PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString())); + Assert.assertArrayEquals("a failed truncate must NOT destroy the dictionary", + before, Files.readAllBytes(f)); + + // And once the filesystem is writable again, the SAME file recovers its + // intact prefix in full -- which is the whole point of not destroying it. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals("the intact prefix survives the transient", 2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("one", s.getQuick(0)); - Assert.assertEquals("two", s.getQuick(1)); - Assert.assertEquals("the torn tail is trimmed once truncate works", cleanLen, Files.size(f)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals("the intact prefix survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + Assert.assertEquals("the torn tail is trimmed once truncate works", cleanLen, Files.size(f)); } }); } - private static void rmDir(Path dir) { - TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); + private Path newFolder(String name) throws IOException { + return temporaryFolder.newFolder(name).toPath(); } - /** * Fails every {@link #truncate(int, long)} -- reproducing a host where the * torn/stale-tail truncate cannot succeed (read-only remount, Windows share From 78a1d3206499e7828d185c885f4baae860323359 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 16 Jul 2026 12:31:16 +0100 Subject: [PATCH 74/78] Fix Windows QWP client test failures --- .../qwp/client/QwpWebSocketSender.java | 32 +++++++++++++++++++ .../qwp/client/sf/cursor/SlotLock.java | 12 ++++--- .../qwp/client/DeltaDictRecoveryTest.java | 30 +++++++---------- .../qwp/client/SelfSufficientFramesTest.java | 21 +++++------- .../qwp/client/sf/cursor/SlotLockTest.java | 6 ++-- 5 files changed, 63 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 696e168e..85cf1148 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1796,6 +1796,38 @@ public long getPendingBytes() { return pendingBytes; } + /** + * Snapshot of the producer's symbol prefix whose persisted-dictionary chunks + * have committed. The persisted size advances only after the chunk CRC and + * payload have been written, so this observes the write-ahead boundary without + * reopening the live mmap-backed dictionary (which would attempt recovery-tail + * truncation and is not supported while the producer owns the file on Windows). + * Returns {@code null} in memory mode or when the persisted dictionary is + * unavailable. + */ + @TestOnly + public ObjList getPersistedSymbolsForTest() { + CursorSendEngine engine = cursorEngine; + if (engine == null) { + return null; + } + PersistedSymbolDict persisted = engine.getPersistedSymbolDict(); + if (persisted == null) { + return null; + } + int persistedSize = persisted.size(); + int globalSize = globalSymbolDictionary.size(); + if (persistedSize > globalSize) { + throw new IllegalStateException("persisted symbol dictionary exceeds producer dictionary" + + " [persisted=" + persistedSize + ", producer=" + globalSize + ']'); + } + ObjList snapshot = new ObjList<>(persistedSize); + for (int i = 0; i < persistedSize; i++) { + snapshot.add(globalSymbolDictionary.getSymbol(i)); + } + return snapshot; + } + /** * Server-advertised cap on the per-batch raw byte size. Zero before the * first connect; updated by every successful reconnect via diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index eb6281a7..0ec9f9f5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -31,6 +31,8 @@ import io.questdb.client.std.Unsafe; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; /** * Advisory exclusive locks for a single SF slot. @@ -101,13 +103,15 @@ public static SlotLock acquire(String slotDir) { */ public static SlotLock acquireLogical(String slotDir) { validateSlotDir(slotDir); - int separator = slotDir.lastIndexOf('/'); - if (separator < 1 || separator == slotDir.length() - 1) { + Path slotPath = Paths.get(slotDir); + Path parentPath = slotPath.getParent(); + Path slotNamePath = slotPath.getFileName(); + if (parentPath == null || slotNamePath == null || slotNamePath.toString().isEmpty()) { throw new IllegalArgumentException( "slotDir must contain a parent and slot name: " + slotDir); } - String parentDir = slotDir.substring(0, separator); - String slotName = slotDir.substring(separator + 1); + String parentDir = parentPath.toString(); + String slotName = slotNamePath.toString(); String logicalLockDir = parentDir + "/" + LOGICAL_LOCK_DIR_NAME; ensureDirectory(logicalLockDir, "logical slot lock dir"); String lockPath = logicalLockDir + "/" + slotName + ".lock"; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 885a3699..14d6c846 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -877,15 +877,11 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception // The persisted dictionary must hold "s0" EXACTLY ONCE. // Pre-fix, the retry duplicated it (size == 2). - PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); - Assert.assertNotNull(pd); - try { - Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol", - 1, pd.size()); - Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); - } finally { - pd.close(); - } + ObjList persisted = ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); + Assert.assertNotNull(persisted); + Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol", + 1, persisted.size()); + Assert.assertEquals("s0", persisted.getQuick(0)); } finally { try { sender.close(); @@ -942,16 +938,12 @@ public void testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating() thr // The else branch persisted ONLY s1 (the suffix). The dictionary holds // s0, s1 exactly once each. Pre-fix (appendSymbols from // sentMaxSymbolId+1) re-appended s0, giving size 3. - PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); - Assert.assertNotNull(pd); - try { - Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix", - 2, pd.size()); - Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); - Assert.assertEquals("s1", pd.readLoadedSymbols().getQuick(1)); - } finally { - pd.close(); - } + ObjList persisted = ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); + Assert.assertNotNull(persisted); + Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix", + 2, persisted.size()); + Assert.assertEquals("s0", persisted.getQuick(0)); + Assert.assertEquals("s1", persisted.getQuick(1)); } finally { try { sender.close(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index c0d27e09..b534d53a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -26,8 +26,8 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.ObjList; -import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -389,18 +389,13 @@ public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception // The write-ahead already ran: both of the batch's new symbols are // durable even though the frame that references them never // published. Move the persist after sealAndSwapBuffer and this is 0. - PersistedSymbolDict pd = PersistedSymbolDict.open( - sfDir.resolve("default").toString()); - Assert.assertNotNull(pd); - try { - Assert.assertEquals("the split path must persist its new symbols BEFORE " - + "publishing the frame that references them", 2, pd.size()); - ObjList symbols = pd.readLoadedSymbols(); - Assert.assertEquals("alpha", symbols.getQuick(0)); - Assert.assertEquals("bravo", symbols.getQuick(1)); - } finally { - pd.close(); - } + ObjList persisted = + ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); + Assert.assertNotNull(persisted); + Assert.assertEquals("the split path must persist its new symbols BEFORE " + + "publishing the frame that references them", 2, persisted.size()); + Assert.assertEquals("alpha", persisted.getQuick(0)); + Assert.assertEquals("bravo", persisted.getQuick(1)); } finally { try { sender.close(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 6643bff8..d38ef9e5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -106,8 +106,10 @@ public void testCloseReleasesLock() throws Exception { @Test public void testLogicalLockRemainsContendedAcrossSlotRenameAndRecreate() throws Exception { TestUtils.assertMemoryLeak(() -> { - String slot = parentDir + "/rename"; - String moved = parentDir + "/rename.quarantined"; + // Use the platform-native separator. In particular, this exercises + // acquireLogical with a backslash-only path on Windows. + String slot = Paths.get(parentDir, "rename").toString(); + String moved = Paths.get(parentDir, "rename.quarantined").toString(); assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); try (SlotLock ignored = SlotLock.acquireLogical(slot)) { From 12a3aec8a70dd7b04903e709a964fe099063eafa Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 16 Jul 2026 13:51:55 +0100 Subject: [PATCH 75/78] Fix QWP store-and-forward recovery safety Retain source rows when split preflight rejects an oversized table. Allow full dictionary frames to re-anchor ACKed recovery gaps while preserving unacked-gap safety. Bump the segment format and install a legacy-reader rollback barrier. --- .../qwp/client/QwpWebSocketSender.java | 1 - .../client/sf/cursor/CursorSendEngine.java | 8 +++ .../qwp/client/sf/cursor/MmapSegment.java | 71 ++++++++++++++++--- .../sf/cursor/RecoveredFrameAnalysis.java | 39 +++++++--- .../qwp/client/sf/cursor/SegmentRing.java | 29 +++++++- .../qwp/client/DeltaDictRecoveryTest.java | 7 +- .../qwp/client/RecoveryReplayTest.java | 4 +- .../qwp/client/SelfSufficientFramesTest.java | 15 ++++ .../sf/cursor/CursorSendEngineTest.java | 51 ++++++++++++- ...CursorWebSocketSendLoopOrphanTailTest.java | 50 +++++++++++++ ...sorWebSocketSendLoopTornDictGuardTest.java | 3 +- .../qwp/client/sf/cursor/MmapSegmentTest.java | 27 +++++++ .../client/test/impl/SenderPoolSfTest.java | 4 +- 13 files changed, 283 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 85cf1148..e0dd4da8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3664,7 +3664,6 @@ private void flushPendingRowsSplit( splitFrameBodyBytes.getQuick(bodyIdx), simBaseline, currentBatchMaxSymbolId); bodyIdx++; if (messageSize > cap) { - resetTableBuffersAfterFlush(keys); throw new LineSenderException("single table batch too large for server batch cap") .put(" [table=").put(tableName) .put(", messageSize=").put(messageSize) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 5fdc129c..38bca269 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -254,6 +254,14 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man PersistedSymbolDict persistedDictInProgress = null; RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { + // v2 segment payloads may depend on a persisted symbol-dictionary + // prefix. Install the rollback barrier before recovery or any new + // append so a v1-only client can never skip the v2 files, treat the + // slot as empty, and silently restart at FSN 0. Current recovery + // recognizes and skips the reserved guard filenames. + if (!memoryMode) { + SegmentRing.installLegacyReaderBarrier(sfDir); + } // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 8942cdfb..f74c8655 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -46,7 +46,7 @@ *

      * On-disk layout — header and frame format: * 

      - *   [u32 magic 'SF01'] [u8 ver=1] [u8 flags=0] [u16 reserved=0]
+ *   [u32 magic 'SF01'] [u8 ver]   [u8 flags=0] [u16 reserved=0]
  *   [u64 baseSeq]       [u64 createdMicros]                       24-byte header
  *   frame, frame, ...                                              each frame:
  *                                                                  [u32 crc32c]
@@ -65,7 +65,8 @@ public final class MmapSegment implements QuietCloseable {
     public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian
     public static final int FRAME_HEADER_SIZE = 8;   // u32 crc + u32 payloadLen
     public static final int HEADER_SIZE = 24;
-    public static final byte VERSION = 1;
+    public static final byte LEGACY_VERSION = 1;
+    public static final byte VERSION = 2;
     private static final int[] CRC32C_TABLE = buildCrc32cTable();
     private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class);
 
@@ -76,6 +77,7 @@ public final class MmapSegment implements QuietCloseable {
     // memory-backed segments — same cursor architecture, no disk involvement.
     // close() and msync() branch on this flag.
     private final boolean memoryBacked;
+    private final byte version;
     // appendCursor: written only by the producer thread, never read by anyone else
     // — it's the reservation cursor. Plain field is fine.
     private long appendCursor;
@@ -106,7 +108,7 @@ public final class MmapSegment implements QuietCloseable {
 
     private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
                         long baseSeq, long initialCursor, long frameCount,
-                        boolean memoryBacked, long tornTailBytes) {
+                        boolean memoryBacked, long tornTailBytes, byte version) {
         this.path = path;
         this.fd = fd;
         this.mmapAddress = mmapAddress;
@@ -117,6 +119,7 @@ private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
         this.frameCount = frameCount;
         this.memoryBacked = memoryBacked;
         this.tornTailBytes = tornTailBytes;
+        this.version = version;
     }
 
     /**
@@ -164,6 +167,50 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long
      * per-call {@code byte[]} + native-malloc the way the String overload does.
      */
     public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) {
+        return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, VERSION);
+    }
+
+    /**
+     * Creates a tiny v1 segment containing one deliberately non-QWP payload.
+     * Two such files with non-contiguous base sequences form the rollback
+     * barrier installed by {@link SegmentRing}: a v1-only reader opens both
+     * and fails its mandatory FSN-contiguity check before replay. If a process
+     * dies after creating only one guard, that reader still adopts a non-empty
+     * log and sends the invalid one-byte head frame first, so it cannot silently
+     * discard the v2 files and start a fresh slot.
+     */
+    static void createLegacyReaderGuard(String path, long baseSeq) {
+        long pathPtr = FilesFacade.INSTANCE.allocNativePath(path);
+        long payload = 0L;
+        try (MmapSegment segment = create(
+                FilesFacade.INSTANCE,
+                pathPtr,
+                path,
+                baseSeq,
+                HEADER_SIZE + FRAME_HEADER_SIZE + 1L,
+                LEGACY_VERSION
+        )) {
+            payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putByte(payload, (byte) 0);
+            if (segment.tryAppend(payload, 1) != HEADER_SIZE) {
+                throw new MmapSegmentException("could not append legacy-reader guard frame: " + path);
+            }
+        } finally {
+            if (payload != 0L) {
+                Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT);
+            }
+            FilesFacade.INSTANCE.freeNativePath(pathPtr);
+        }
+    }
+
+    private static MmapSegment create(
+            FilesFacade ff,
+            long pathPtr,
+            String displayPath,
+            long baseSeq,
+            long sizeBytes,
+            byte version
+    ) {
         if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) {
             throw new IllegalArgumentException(
                     "sizeBytes too small for header + one minimal frame: " + sizeBytes);
@@ -194,12 +241,13 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat
             }
             // Header goes straight into the mapping — no separate write syscall.
             Unsafe.getUnsafe().putInt(addr, FILE_MAGIC);
-            Unsafe.getUnsafe().putByte(addr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(addr + 4, version);
             Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
+            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, false, 0L, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
@@ -236,7 +284,8 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0);
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
+            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, true, 0L, VERSION);
         } catch (Throwable t) {
             Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT);
             throw t;
@@ -300,7 +349,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                         "bad magic in " + path + ": 0x" + Integer.toHexString(magic));
             }
             byte version = Unsafe.getUnsafe().getByte(addr + 4);
-            if (version != VERSION) {
+            if (version != LEGACY_VERSION && version != VERSION) {
                 throw new MmapSegmentException("unsupported version in " + path + ": " + version);
             }
             long baseSeq = Unsafe.getUnsafe().getLong(addr + 8);
@@ -326,7 +375,8 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                                 + "Investigate disk health or unexpected writer crash.",
                         path, tornTail, lastGood, fileSize, count);
             }
-            return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail);
+            return new MmapSegment(path, fd, addr, fileSize, baseSeq,
+                    lastGood, count, false, tornTail, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
@@ -440,6 +490,11 @@ public long sizeBytes() {
         return sizeBytes;
     }
 
+    /** On-disk format version read from or written to this segment. */
+    public byte version() {
+        return version;
+    }
+
     /**
      * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]}
      * starting at the current append cursor, then advances both cursors
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
index c9650cfd..386d0f1e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -41,6 +41,7 @@
 final class RecoveredFrameAnalysis implements QuietCloseable {
 
     private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8;
+    private final long ackedFsn;
     private final int baseline;
     private long committedBoundaryFsn = -1L;
     private long committedCoverage;
@@ -51,6 +52,7 @@ final class RecoveredFrameAnalysis implements QuietCloseable {
     private int committedRawLen;
     private long framesVisited;
     private boolean runningGap;
+    private boolean runningUnackedGap;
     private long runningMaxDeltaEnd;
     private long runningMaxDeltaStart;
     private long rawAddr;
@@ -59,8 +61,9 @@ final class RecoveredFrameAnalysis implements QuietCloseable {
     private int runningRawLen;
     private long runningCoverage;
 
-    RecoveredFrameAnalysis(int baseline) {
+    RecoveredFrameAnalysis(int baseline, long ackedFsn) {
         this.baseline = baseline;
+        this.ackedFsn = ackedFsn;
         this.runningCoverage = baseline;
         this.committedCoverage = baseline;
     }
@@ -74,7 +77,7 @@ void accept(long fsn, long payload, int payloadLen) {
                 ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS)
                 : 0;
         if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) {
-            foldDelta(payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
+            foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
         }
 
         // Only a positively identified deferred QWP frame can belong to an
@@ -179,10 +182,10 @@ private void appendRaw(long entryStart, int entryLen) {
         runningRawCount++;
     }
 
-    private void foldDelta(long p, long limit) {
+    private void foldDelta(long fsn, long p, long limit) {
         long encodedStart = readVarint(p, limit);
         if (encodedStart < 0L) {
-            runningGap = true;
+            markGap(fsn);
             return;
         }
         int startLen = (int) (encodedStart & 7L);
@@ -194,7 +197,7 @@ private void foldDelta(long p, long limit) {
 
         long encodedCount = readVarint(p, limit);
         if (encodedCount < 0L) {
-            runningGap = true;
+            markGap(fsn);
             return;
         }
         int countLen = (int) (encodedCount & 7L);
@@ -207,10 +210,21 @@ private void foldDelta(long p, long limit) {
             runningMaxDeltaEnd = deltaEnd;
         }
         if (runningGap) {
-            return;
+            // A full dictionary is a new self-sufficient epoch, but it may only
+            // repair a gap that is entirely behind the durable ACK watermark.
+            // If any gapped frame will replay first, accepting this reset would
+            // hide the unsafe wire-order gap and let the server observe missing
+            // ids before it reaches the full frame.
+            if (deltaStart != 0L || runningUnackedGap) {
+                return;
+            }
+            runningGap = false;
+            runningCoverage = baseline;
+            runningRawLen = 0;
+            runningRawCount = 0;
         }
         if (deltaStart > runningCoverage) {
-            runningGap = true;
+            markGap(fsn);
             return;
         }
 
@@ -219,14 +233,14 @@ private void foldDelta(long p, long limit) {
             long entryStart = p;
             long encodedLen = readVarint(p, limit);
             if (encodedLen < 0L) {
-                runningGap = true;
+                markGap(fsn);
                 return;
             }
             int varintLen = (int) (encodedLen & 7L);
             long symbolLen = encodedLen >>> 3;
             p += varintLen;
             if (symbolLen > limit - p) {
-                runningGap = true;
+                markGap(fsn);
                 return;
             }
             p += symbolLen;
@@ -239,6 +253,13 @@ private void foldDelta(long p, long limit) {
         }
     }
 
+    private void markGap(long fsn) {
+        runningGap = true;
+        if (fsn > ackedFsn) {
+            runningUnackedGap = true;
+        }
+    }
+
     /**
      * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an
      * unterminated/oversized 32-bit protocol varint.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index 8e596a01..5ba787f3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -62,6 +62,8 @@ public final class SegmentRing implements QuietCloseable {
     public static final long BACKPRESSURE_NO_SPARE = -1L;
     /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */
     public static final long PAYLOAD_TOO_LARGE = -2L;
+    static final String LEGACY_READER_GUARD_A = ".qwp-v2-guard-a.sfa";
+    static final String LEGACY_READER_GUARD_B = ".qwp-v2-guard-b.sfa";
     private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class);
     // Tally of baseSeq comparisons performed by sortByBaseSeq across every
     // openExisting() recovery on this JVM. Used by SegmentRingTest to
@@ -178,7 +180,9 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                 int rc = 1;
                 while (rc > 0) {
                     String name = Files.utf8ToString(Files.findName(find));
-                    if (name != null && name.endsWith(".sfa")) {
+                    if (name != null
+                            && name.endsWith(".sfa")
+                            && !isLegacyReaderGuard(name)) {
                         String path = sfDir + "/" + name;
                         MmapSegment seg = null;
                         try {
@@ -325,6 +329,23 @@ public long ackedFsn() {
         return ackedFsn;
     }
 
+    /**
+     * Installs an on-disk rollback barrier before any v2 segment can be
+     * created or appended. The guards are valid v1 segments so a legacy
+     * reader cannot skip them as an unknown format. Their deliberately
+     * non-contiguous FSN ranges make its recovery fail closed before replay;
+     * current readers recognize the reserved names and omit them from the
+     * logical ring.
+     */
+    static void installLegacyReaderBarrier(String sfDir) {
+        MmapSegment.createLegacyReaderGuard(
+                sfDir + '/' + LEGACY_READER_GUARD_A,
+                Long.MAX_VALUE - 4L);
+        MmapSegment.createLegacyReaderGuard(
+                sfDir + '/' + LEGACY_READER_GUARD_B,
+                Long.MAX_VALUE - 1L);
+    }
+
     /**
      * I/O thread (or anyone tracking ACK) advances the ACK cursor. {@code seq}
      * is cumulative -- the server has confirmed every FSN up to and including
@@ -586,7 +607,7 @@ public synchronized long collectReplaySymbolsAbove(
      * active segment. The returned native suffix remains owned by the caller.
      */
     synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) {
-        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline);
+        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline, ackedFsn);
         try {
             for (int i = 0, n = sealedSegments.size(); i < n; i++) {
                 sealedSegments.get(i).scanRecovery(analysis);
@@ -859,6 +880,10 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) {
         }
     }
 
+    private static boolean isLegacyReaderGuard(String name) {
+        return LEGACY_READER_GUARD_A.equals(name) || LEGACY_READER_GUARD_B.equals(name);
+    }
+
     private static void swap(ObjList list, int i, int j) {
         if (i == j) return;
         MmapSegment tmp = list.get(i);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
index 14d6c846..10d4ef32 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
@@ -414,7 +414,8 @@ private static int countSegmentFiles(java.nio.file.Path dir) {
         int n = 0;
         if (files != null) {
             for (java.io.File f : files) {
-                if (f.getName().endsWith(".sfa")) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")) {
                     n++;
                 }
             }
@@ -1495,7 +1496,9 @@ private static boolean hasSegmentFile(java.nio.file.Path dir) {
         java.io.File[] files = dir.toFile().listFiles();
         if (files != null) {
             for (java.io.File f : files) {
-                if (f.getName().endsWith(".sfa") && f.length() > 0) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")
+                        && f.length() > 0) {
                     return true;
                 }
             }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
index e868dafc..274d39bf 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
@@ -154,7 +154,9 @@ private static int countPopulatedSegmentFiles(String dir) {
             int rc = 1;
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
-                if (name != null && name.endsWith(".sfa")) {
+                if (name != null
+                        && name.endsWith(".sfa")
+                        && !name.startsWith(".qwp-v2-guard-")) {
                     try {
                         try (MmapSegment seg = MmapSegment.openExisting(dir + "/" + name)) {
                             if (seg.frameCount() > 0) n++;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
index b534d53a..0c88bc11 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
@@ -316,6 +316,21 @@ public void testOversizedTableSplitStrandsNothing() throws Exception {
                         Assert.assertTrue(e.getMessage(),
                                 e.getMessage().contains("too large for server batch cap"));
                     }
+                    // Sender.flush() is retryable: a rejected flush must leave the
+                    // source rows intact. Repeating it under the same cap therefore
+                    // has to reject the same batch again. Before the fix, the first
+                    // failure reset every table buffer and this second flush was a
+                    // silent no-op -- the caller's rows had been lost.
+                    try {
+                        sender.flush();
+                        Assert.fail("retrying the retained oversized batch must throw again");
+                    } catch (LineSenderException e) {
+                        Assert.assertTrue(e.getMessage(),
+                                e.getMessage().contains("too large for server batch cap"));
+                    }
+                    // Explicit reset is the caller-controlled discard boundary and
+                    // prevents close() from making a third retry in this test.
+                    sender.reset();
                     // close() drains the ring: pre-fix, the stranded "small" frame
                     // would be sent (and committed) here.
                 }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
index 0e874ba5..0bdddb3e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
@@ -298,6 +298,51 @@ public void testRecoveryAdvancesAckedFsnPastWatermark() throws Exception {
         });
     }
 
+    @Test
+    public void testV2DiskSlotForcesLegacyReaderToFailClosed() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+            try {
+                try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) {
+                    assertEquals(0L, engine.appendBlocking(buf, 16));
+                }
+
+                // Real segments carry v2: their payload may depend on the
+                // persisted dictionary and must never be replayed by a v1-only
+                // client as though every frame were self-sufficient.
+                try (MmapSegment data = MmapSegment.openExisting(tmpDir + "/sf-initial.sfa")) {
+                    assertEquals(MmapSegment.VERSION, data.version());
+                    assertEquals(1L, data.frameCount());
+                }
+
+                // A legacy reader rejects the v2 data segment, but accepts both
+                // guards as ordinary non-empty v1 segments. Their ranges cannot
+                // be contiguous, so its mandatory global recovery check throws
+                // before any replay or fresh-slot fallback can happen.
+                try (MmapSegment guardA = MmapSegment.openExisting(
+                        tmpDir + "/.qwp-v2-guard-a.sfa");
+                     MmapSegment guardB = MmapSegment.openExisting(
+                             tmpDir + "/.qwp-v2-guard-b.sfa")) {
+                    assertEquals(MmapSegment.LEGACY_VERSION, guardA.version());
+                    assertEquals(MmapSegment.LEGACY_VERSION, guardB.version());
+                    assertEquals(1L, guardA.frameCount());
+                    assertEquals(1L, guardB.frameCount());
+                    assertTrue("legacy guard ranges must force an FSN gap",
+                            guardA.baseSeq() + guardA.frameCount() != guardB.baseSeq());
+                }
+
+                // Current recovery skips the reserved guards and adopts only
+                // the real v2 log.
+                try (CursorSendEngine recovered = new CursorSendEngine(tmpDir, 4096)) {
+                    assertTrue(recovered.wasRecoveredFromDisk());
+                    assertEquals(0L, recovered.publishedFsn());
+                }
+            } finally {
+                Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testRecoveryIgnoresWatermarkAbovePublishedFsn() throws Exception {
         // Pin the corruption defence: a watermark higher than what the
@@ -413,7 +458,11 @@ public void testRestartIntoNonEmptySfDirContinuesFsnSequence() throws Exception
                     int rc = 1;
                     while (rc > 0) {
                         String name = Files.utf8ToString(Files.findName(find));
-                        if (name != null && name.endsWith(".sfa")) sfaCount++;
+                        if (name != null
+                                && name.endsWith(".sfa")
+                                && !name.startsWith(".qwp-v2-guard-")) {
+                            sfaCount++;
+                        }
                         rc = Files.findNext(find);
                     }
                 } finally {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
index 52235637..c96bfefe 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
@@ -28,6 +28,7 @@
 import io.questdb.client.cutlass.http.client.WebSocketClient;
 import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
 import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
 import io.questdb.client.network.PlainSocketFactory;
@@ -398,6 +399,55 @@ public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exce
         });
     }
 
+    @Test
+    public void testSelfSufficientFrameRepairsAckedRecoveryGap() throws Exception {
+        // fsn 0 models the tail of an old delta epoch whose registering frames
+        // have already been trimmed: with an empty persisted dictionary its
+        // deltaStart=1 is a gap. It is durably ACKed, so it will not replay.
+        // fsn 1 starts a new, self-sufficient epoch from id 0 and is the first
+        // frame that WILL replay. Recovery must use that full frame as the new
+        // source of truth instead of permanently latching the earlier gap.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                appendDeltaFrame(engine, false, 1, 0);
+                appendDeltaSymbolFrame(engine, 0, 'a');
+            }
+            try (AckWatermark watermark = AckWatermark.open(tmpDir)) {
+                assertNotNull(watermark);
+                watermark.write(0L);
+            }
+
+            try (CursorSendEngine engine = newEngine()) {
+                assertEquals(0L, engine.ackedFsn());
+                ObjList recovered = new ObjList<>();
+                assertEquals("the full frame must re-anchor replay coverage",
+                        1L, engine.collectReplaySymbolsAbove(0, recovered));
+                assertEquals(1, recovered.size());
+                assertEquals("a", recovered.getQuick(0));
+            }
+        });
+    }
+
+    @Test
+    public void testSelfSufficientFrameCannotHideUnackedRecoveryGap() throws Exception {
+        // Safety twin: when the gapped frame itself is unacked, it reaches the
+        // wire before the later full frame. Recovery must keep the gap latched
+        // and quarantine rather than pretending the later reset repairs the
+        // invalid replay order.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                appendDeltaFrame(engine, false, 1, 0);
+                appendDeltaSymbolFrame(engine, 0, 'a');
+            }
+            try (CursorSendEngine engine = newEngine()) {
+                ObjList recovered = new ObjList<>();
+                assertEquals("an unacked gap remains unreplayable",
+                        -1L, engine.collectReplaySymbolsAbove(0, recovered));
+                assertEquals(0, recovered.size());
+            }
+        });
+    }
+
     // ---------------------------------------------------------------------
     // harness
     // ---------------------------------------------------------------------
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
index 7dabc741..5a666dc7 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
@@ -119,7 +119,8 @@ private static int countSegmentFiles(Path dir) {
         int n = 0;
         if (files != null) {
             for (File f : files) {
-                if (f.getName().endsWith(".sfa")) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")) {
                     n++;
                 }
             }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
index 177ee5f6..4707c233 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
@@ -35,6 +35,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.RandomAccessFile;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -113,6 +115,31 @@ public void testCreateAppendCloseReopenScansAllFrames() throws Exception {
         });
     }
 
+    @Test
+    public void testCurrentReaderStillOpensLegacyV1Segment() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            String path = tmpDir + "/seg-v1.sfa";
+            long buf = Unsafe.malloc(8, MemoryTag.NATIVE_DEFAULT);
+            try {
+                try (MmapSegment seg = MmapSegment.create(path, 7L, 4096)) {
+                    assertEquals(MmapSegment.VERSION, seg.version());
+                    assertTrue(seg.tryAppend(buf, 8) >= 0L);
+                }
+                try (RandomAccessFile file = new RandomAccessFile(path, "rw")) {
+                    file.seek(4L);
+                    file.writeByte(MmapSegment.LEGACY_VERSION);
+                }
+                try (MmapSegment legacy = MmapSegment.openExisting(path)) {
+                    assertEquals(MmapSegment.LEGACY_VERSION, legacy.version());
+                    assertEquals(7L, legacy.baseSeq());
+                    assertEquals(1L, legacy.frameCount());
+                }
+            } finally {
+                Unsafe.free(buf, 8, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception {
         TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
index cfef2feb..2f213c3a 100644
--- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
+++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
@@ -1847,7 +1847,9 @@ private static boolean hasSegmentFile(String slotPath) {
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
                 rc = Files.findNext(find);
-                if (name != null && name.endsWith(".sfa")) {
+                if (name != null
+                        && name.endsWith(".sfa")
+                        && !name.startsWith(".qwp-v2-guard-")) {
                     return true;
                 }
             }

From 350e4a397c7134a817b013fb7b19265141a410fe Mon Sep 17 00:00:00 2001
From: glasstiger 
Date: Thu, 16 Jul 2026 14:17:23 +0100
Subject: [PATCH 76/78] Restore QWP API compatibility

Restore the previous QwpWebSocketSender connect overload and the
BackgroundDrainer constructor. Delegate both to the new catch-up dwell
defaults.

Use a saturating conversion for catch-up dwell milliseconds so large
valid values cannot wrap negative and quarantine slots prematurely.
---
 .../qwp/client/QwpWebSocketSender.java        | 39 ++++++++
 .../client/sf/cursor/BackgroundDrainer.java   | 25 +++++
 .../sf/cursor/CursorWebSocketSendLoop.java    |  7 +-
 .../client/QwpPublicApiCompatibilityTest.java | 96 +++++++++++++++++++
 ...WebSocketSendLoopCatchUpAlignmentTest.java | 27 ++++++
 5 files changed, 193 insertions(+), 1 deletion(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index e0dd4da8..f24b1ab1 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -702,6 +702,45 @@ public static QwpWebSocketSender connect(
                 CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
     }
 
+    /**
+     * Compatibility overload retained for callers compiled before the
+     * catch-up cap-gap dwell was added to the master signature.
+     */
+    public static QwpWebSocketSender connect(
+            List endpoints,
+            ClientTlsConfiguration tlsConfig,
+            int autoFlushRows,
+            int autoFlushBytes,
+            long autoFlushIntervalNanos,
+            String authorizationHeader,
+            boolean requestDurableAck,
+            CursorSendEngine cursorEngine,
+            long closeFlushTimeoutMillis,
+            long reconnectMaxDurationMillis,
+            long reconnectInitialBackoffMillis,
+            long reconnectMaxBackoffMillis,
+            Sender.InitialConnectMode initialConnectMode,
+            SenderErrorHandler errorHandler,
+            int errorInboxCapacity,
+            long durableAckKeepaliveIntervalMillis,
+            long authTimeoutMs,
+            int connectTimeoutMs,
+            SenderConnectionListener connectionListener,
+            int connectionListenerInboxCapacity,
+            int maxFrameRejections,
+            long poisonMinEscalationWindowMillis
+    ) {
+        return connect(endpoints, tlsConfig, autoFlushRows, autoFlushBytes,
+                autoFlushIntervalNanos, authorizationHeader, requestDurableAck,
+                cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis,
+                reconnectInitialBackoffMillis, reconnectMaxBackoffMillis,
+                initialConnectMode, errorHandler, errorInboxCapacity,
+                durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs,
+                connectionListener, connectionListenerInboxCapacity,
+                maxFrameRejections, poisonMinEscalationWindowMillis,
+                CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
+    }
+
     /**
      * Master connect overload — also accepts the poison-frame detector
      * threshold ({@code max_frame_rejections}): consecutive server-active
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
index 8e287b4a..38b6b036 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
@@ -160,6 +160,31 @@ public BackgroundDrainer(
                 CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
     }
 
+    /**
+     * Compatibility constructor retained for callers compiled before the
+     * catch-up cap-gap dwell was added to the master signature.
+     */
+    public BackgroundDrainer(
+            String slotPath,
+            long segmentSizeBytes,
+            long sfMaxTotalBytes,
+            CursorWebSocketSendLoop.ReconnectFactory clientFactory,
+            long reconnectMaxDurationMillis,
+            long reconnectInitialBackoffMillis,
+            long reconnectMaxBackoffMillis,
+            boolean requestDurableAck,
+            long durableAckKeepaliveIntervalMillis,
+            int maxHeadFrameRejections,
+            long poisonMinEscalationWindowMillis
+    ) {
+        this(slotPath, segmentSizeBytes, sfMaxTotalBytes, clientFactory,
+                reconnectMaxDurationMillis, reconnectInitialBackoffMillis,
+                reconnectMaxBackoffMillis, requestDurableAck,
+                durableAckKeepaliveIntervalMillis, maxHeadFrameRejections,
+                poisonMinEscalationWindowMillis,
+                CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
+    }
+
     /**
      * Master constructor — also accepts the poison-frame detector threshold
      * ({@code max_frame_rejections}) forwarded to the drain loop's
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 5c59823c..1f1dc034 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -53,6 +53,7 @@
 import java.util.Arrays;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.LockSupport;
 
@@ -682,8 +683,12 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
                     "catchUpCapGapMinEscalationWindowMillis must be >= 0: "
                             + catchUpCapGapMinEscalationWindowMillis);
         }
+        // TimeUnit conversion saturates at Long.MAX_VALUE. A raw multiply
+        // wraps large valid millisecond values negative, which makes the
+        // elapsed-time gate appear satisfied as soon as the strike threshold
+        // is reached and can quarantine a recoverable orphan slot prematurely.
         this.catchUpCapGapMinEscalationWindowNanos =
-                catchUpCapGapMinEscalationWindowMillis * 1_000_000L;
+                TimeUnit.MILLISECONDS.toNanos(catchUpCapGapMinEscalationWindowMillis);
         if (catchUpCapGapPolicy == null) {
             throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null");
         }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java
new file mode 100644
index 00000000..d2770145
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java
@@ -0,0 +1,96 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.ClientTlsConfiguration;
+import io.questdb.client.Sender;
+import io.questdb.client.SenderConnectionListener;
+import io.questdb.client.SenderErrorHandler;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import org.junit.Test;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class QwpPublicApiCompatibilityTest {
+
+    @Test
+    public void testBackgroundDrainerLegacyConstructorDescriptorIsPreserved() throws Exception {
+        Constructor constructor = BackgroundDrainer.class.getConstructor(
+                String.class,
+                long.class,
+                long.class,
+                CursorWebSocketSendLoop.ReconnectFactory.class,
+                long.class,
+                long.class,
+                long.class,
+                boolean.class,
+                long.class,
+                int.class,
+                long.class
+        );
+        assertTrue(Modifier.isPublic(constructor.getModifiers()));
+    }
+
+    @Test
+    public void testQwpWebSocketSenderLegacyConnectDescriptorIsPreserved() throws Exception {
+        Method method = QwpWebSocketSender.class.getMethod(
+                "connect",
+                List.class,
+                ClientTlsConfiguration.class,
+                int.class,
+                int.class,
+                long.class,
+                String.class,
+                boolean.class,
+                CursorSendEngine.class,
+                long.class,
+                long.class,
+                long.class,
+                long.class,
+                Sender.InitialConnectMode.class,
+                SenderErrorHandler.class,
+                int.class,
+                long.class,
+                long.class,
+                int.class,
+                SenderConnectionListener.class,
+                int.class,
+                int.class,
+                long.class
+        );
+        assertEquals(QwpWebSocketSender.class, method.getReturnType());
+        assertTrue(Modifier.isPublic(method.getModifiers()));
+        assertTrue(Modifier.isStatic(method.getModifiers()));
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
index 34ef3c4a..acc8102a 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
@@ -397,6 +397,33 @@ public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() t
         });
     }
 
+    @Test
+    public void testCatchUpCapGapDwellConversionSaturatesInsteadOfOverflowing() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            long maxExactMillis = Long.MAX_VALUE / 1_000_000L;
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop exactLoop = newLoop(
+                        engine, new CatchUpCapturingClient(0), maxExactMillis);
+                try {
+                    assertEquals(maxExactMillis * 1_000_000L,
+                            readLong(exactLoop, "catchUpCapGapMinEscalationWindowNanos"));
+                } finally {
+                    exactLoop.close();
+                }
+
+                CursorWebSocketSendLoop saturatedLoop = newLoop(
+                        engine, new CatchUpCapturingClient(0), maxExactMillis + 1L);
+                try {
+                    assertEquals("an oversized dwell must become effectively infinite, not negative",
+                            Long.MAX_VALUE,
+                            readLong(saturatedLoop, "catchUpCapGapMinEscalationWindowNanos"));
+                } finally {
+                    saturatedLoop.close();
+                }
+            }
+        });
+    }
+
     @Test
     public void testCapGapEpisodeWithANegativeAnchorStillEscalates() throws Exception {
         // A cap-gap episode anchored at a NEGATIVE nanoTime instant must escalate like any

From a341a9b70f850335d00f55f3a3b70c9cfcbed019 Mon Sep 17 00:00:00 2001
From: glasstiger 
Date: Thu, 16 Jul 2026 14:59:28 +0100
Subject: [PATCH 77/78] Optimize QWP symbol dictionary recovery

---
 .../qwp/client/QwpWebSocketSender.java        |  19 +-
 .../client/sf/cursor/CursorSendEngine.java    |  24 ++
 .../sf/cursor/CursorWebSocketSendLoop.java    |  91 +++--
 .../qwp/client/sf/cursor/MmapSegment.java     |  45 ++-
 .../client/sf/cursor/PersistedSymbolDict.java | 333 +++++++++++-------
 .../sf/cursor/RecoveredFrameAnalysis.java     |  28 +-
 ...WebSocketSendLoopCatchUpAlignmentTest.java |   1 +
 ...CursorWebSocketSendLoopMirrorLeakTest.java |  41 ++-
 ...CursorWebSocketSendLoopOrphanTailTest.java |  31 ++
 .../qwp/client/sf/cursor/MmapSegmentTest.java |   2 +
 .../sf/cursor/PersistedSymbolDictTest.java    |   7 +
 11 files changed, 410 insertions(+), 212 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index f24b1ab1..1fba1452 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -3942,7 +3942,7 @@ private void resetSymbolDictStateForNewConnection() {
      * 
        
      *   
      1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
        2. 
      *   
      2. the surviving frames' OWN delta sections, for every id above that prefix
-     *       ({@link CursorSendEngine#collectReplaySymbolsAbove}).
        3. 
+     *       ({@link CursorSendEngine#addRecoveredSymbolsTo}).
      * 
      
      * Those are exactly the two sources, in exactly the order, that the send loop's mirror
      * is built from: its constructor seeds {@code sentDictCount} from the same dictionary,
@@ -3976,7 +3976,7 @@ private void resetSymbolDictStateForNewConnection() {
      * What still fails clean. A genuine GAP: the ids below a surviving frame's delta
      * start were introduced by frames that were acked and TRIMMED away, so they lived only in
      * the lost dictionary and nothing can rebuild them.
-     * {@code collectReplaySymbolsAbove} returns -1 for that and we throw. It is the same
+     * {@code addRecoveredSymbolsTo} returns -1 for that and we throw. It is the same
      * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so
      * producer and drainer now agree on exactly which slots are recoverable, instead of the
      * producer rejecting slots the drainer drains.
@@ -3990,17 +3990,13 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) {
         //    mirror also seeds sentDictCount from.
         int baseline = 0;
         if (pd != null && pd.size() > 0) {
-            ObjList persisted = pd.readLoadedSymbols();
-            for (int i = 0, n = persisted.size(); i < n; i++) {
-                globalSymbolDictionary.addRecoveredSymbol(persisted.getQuick(i));
-            }
+            pd.addLoadedSymbolsTo(globalSymbolDictionary);
             baseline = globalSymbolDictionary.size();
         }
         // 2. Everything the surviving frames define above that prefix, straight out of their
         //    own delta sections -- the same bytes, in the same order, accumulateSentDict will
         //    feed the mirror as those frames go back on the wire.
-        ObjList fromFrames = new ObjList<>();
-        long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames);
+        long coverage = cursorEngine.addRecoveredSymbolsTo(baseline, globalSymbolDictionary);
         if (coverage < 0) {
             // A gap: the surviving frames reference ids below their own delta start,
             // introduced by frames since acked and trimmed away, and the persisted
@@ -4013,8 +4009,8 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) {
             // "resend required" for delivered data AND -- because such a slot is fully
             // drained -- let build()'s connect-path close unlink the (already-delivered)
             // bytes the quarantine claims to preserve. So DON'T throw: seed the intact
-            // prefix only, leaving fromFrames unused exactly as the send loop's mirror
-            // does on a -1 (it adds nothing), so the producer baseline and the mirror's
+            // prefix only; addRecoveredSymbolsTo adds nothing on a -1 exactly as the
+            // send loop's mirror does, so the producer baseline and the mirror's
             // sentDictCount still agree by construction. The producer resumes above the
             // prefix and the fully-drained slot is cleaned up on close.
             long ackedFsn = cursorEngine.ackedFsn();
@@ -4040,9 +4036,6 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) {
                             + "holds them; the recovered dictionary holds only "
                             + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data");
         }
-        for (int i = 0, n = fromFrames.size(); i < n; i++) {
-            globalSymbolDictionary.addRecoveredSymbol(fromFrames.getQuick(i));
-        }
         // Producer baseline == the coverage the replay will establish == the mirror's
         // sentDictCount once those frames have gone out. The first new frame therefore
         // starts its delta exactly at the tip, and the replay guard passes.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index 38bca269..597c2bc7 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.client.sf.cursor;
 
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.std.Compat;
 import io.questdb.client.std.Files;
@@ -760,6 +761,24 @@ public long collectReplaySymbolsAbove(int baseline, ObjList out) {
         );
     }
 
+    /**
+     * Decodes the cached recovery suffix directly into the producer's global
+     * dictionary. Recovery always builds the analysis with the persisted
+     * prefix size as its baseline, so no intermediate cardinality-sized list is
+     * needed on the production path.
+     */
+    public long addRecoveredSymbolsTo(int baseline, GlobalSymbolDictionary target) {
+        if (recoveredFrameAnalysis == null) {
+            return baseline;
+        }
+        RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline);
+        long coverage = analysis.coverage();
+        if (coverage >= 0L) {
+            analysis.addDecodedSymbolsTo(target);
+        }
+        return coverage;
+    }
+
     long recoveredSymbolCoverage(int baseline) {
         return checkedRecoveryAnalysis(baseline).coverage();
     }
@@ -785,6 +804,11 @@ public long recoveryFramesVisited() {
         return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited();
     }
 
+    @TestOnly
+    public long recoverySymbolEntriesVisited() {
+        return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.symbolEntriesVisited();
+    }
+
     private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) {
         if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) {
             throw new IllegalStateException("recovery symbol baseline mismatch [expected="
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 1f1dc034..3d33bfd0 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -299,6 +299,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
     private long sentDictBytesAddr;
     private int sentDictBytesCapacity;
     private int sentDictBytesLen;
+    // False only while an orphan-drainer loop borrows the persisted prefix.
+    // Any growth first performs a copy-on-write; only owned buffers are freed.
+    private boolean sentDictBytesOwned;
     private int sentDictCount;
     // Reusable staging frame for dictionary catch-up chunks. A reconnect may split a
     // large dictionary into many chunks, and every later reconnect repeats that split;
@@ -721,24 +724,17 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
         // tracks. When the dictionary could not be opened, pd is null, the mirror
         // simply starts empty and grows from the frames themselves.
         PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+        int persistedPrefixLen = 0;
         if (pd != null && pd.size() > 0) {
             int len = pd.loadedEntriesLen();
             if (len > 0) {
-                // COPY the persisted dictionary's loaded-entries buffer into this
-                // loop's own mirror rather than taking ownership of it. The engine
-                // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan
-                // drainer path: BackgroundDrainer builds a fresh send loop per wire
-                // session against the same engine on a durable-ack capability-gap
-                // recycle. A one-shot ownership transfer would leave every loop
-                // after the first with an EMPTY mirror -- it would then send no
-                // reconnect catch-up, and the first replayed delta frame
-                // (deltaStart > 0) would trip the torn-dict guard, falsely
-                // quarantining a healthy slot. Copying keeps the dictionary's
-                // loaded entries intact for the engine's lifetime so every
-                // recycled loop re-seeds; pd.close() (at engine close) frees the
-                // dictionary's copy, this loop frees its own copy on exit.
-                sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
-                Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len);
+                // Seed by reference. The foreground loop takes ownership after
+                // construction succeeds; orphan-drainer loops keep borrowing because
+                // one engine can create several sessions during capability-gap
+                // recycling. A borrowed mirror performs copy-on-write if a recovered
+                // frame suffix must extend it.
+                persistedPrefixLen = len;
+                sentDictBytesAddr = pd.loadedEntriesAddr();
                 sentDictBytesCapacity = len;
                 sentDictBytesLen = len;
                 // Set the count only alongside the bytes so sentDictCount can
@@ -786,16 +782,25 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
                     }
                 }
             } catch (Throwable t) {
-                if (sentDictBytesAddr != 0) {
-                    Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
-                    sentDictBytesAddr = 0;
-                    sentDictBytesCapacity = 0;
-                    sentDictBytesLen = 0;
-                    sentDictCount = 0;
-                }
+                releaseSentDictBytes();
                 throw t;
             }
         }
+        // QwpWebSocketSender seeds the producer dictionary before constructing
+        // its one foreground loop, so that loop can own the recovered prefix and
+        // eliminate the engine-side duplicate. BackgroundDrainer must retain the
+        // prefix for later recycled loops and therefore keeps borrowing it.
+        if (persistedPrefixLen > 0 && catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) {
+            long persistedPrefixAddr = pd.takeLoadedEntries();
+            if (sentDictBytesOwned) {
+                // Copy-on-grow already produced the combined mirror. Retire the
+                // no-longer-needed persisted prefix after successful construction.
+                Unsafe.free(persistedPrefixAddr, persistedPrefixLen, MemoryTag.NATIVE_DEFAULT);
+            } else {
+                assert persistedPrefixAddr == sentDictBytesAddr;
+                sentDictBytesOwned = true;
+            }
+        }
         this.fsnAtZero = fsnAtZero;
         this.parkNanos = parkNanos;
         this.reconnectFactory = reconnectFactory;
@@ -1131,15 +1136,7 @@ public synchronized void close() {
         // thread may still be mid-send, so touching the mirror here would race.
         // A duplicate close observes sentDictBytesAddr == 0 and skips.
         if (loopNeverRan && sentDictBytesAddr != 0) {
-            Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
-            sentDictBytesAddr = 0;
-            sentDictBytesCapacity = 0;
-            sentDictBytesLen = 0;
-            // Reset the count alongside the buffer so the mirror stays all-or-
-            // nothing: a hypothetical close()-then-start() (start() has no closed
-            // guard) must not observe a non-zero sentDictCount against a freed
-            // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up.
-            sentDictCount = 0;
+            releaseSentDictBytes();
         }
         if (loopNeverRan) {
             freeCatchUpFrameBuffer();
@@ -2048,11 +2045,7 @@ private void ioLoop() {
             // The symbol-dict mirror is I/O-thread-owned; free it here, on the
             // owning thread's exit path, after the last send that could touch it.
             if (sentDictBytesAddr != 0) {
-                Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
-                sentDictBytesAddr = 0;
-                sentDictBytesCapacity = 0;
-                sentDictBytesLen = 0;
-                sentDictCount = 0; // keep the mirror all-or-nothing (see close())
+                releaseSentDictBytes();
             }
             freeCatchUpFrameBuffer();
             shutdownLatch.countDown();
@@ -2438,10 +2431,34 @@ private void ensureSentDictCapacity(long required) {
         if (newCap > MAX_SENT_DICT_BYTES) {
             newCap = MAX_SENT_DICT_BYTES;
         }
-        sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+        if (sentDictBytesOwned) {
+            sentDictBytesAddr = Unsafe.realloc(
+                    sentDictBytesAddr,
+                    sentDictBytesCapacity,
+                    (int) newCap,
+                    MemoryTag.NATIVE_DEFAULT);
+        } else {
+            long newAddr = Unsafe.malloc((int) newCap, MemoryTag.NATIVE_DEFAULT);
+            if (sentDictBytesLen > 0) {
+                Unsafe.getUnsafe().copyMemory(sentDictBytesAddr, newAddr, sentDictBytesLen);
+            }
+            sentDictBytesAddr = newAddr;
+            sentDictBytesOwned = true;
+        }
         sentDictBytesCapacity = (int) newCap;
     }
 
+    private void releaseSentDictBytes() {
+        if (sentDictBytesAddr != 0 && sentDictBytesOwned) {
+            Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
+        }
+        sentDictBytesAddr = 0;
+        sentDictBytesCapacity = 0;
+        sentDictBytesLen = 0;
+        sentDictBytesOwned = false;
+        sentDictCount = 0;
+    }
+
     private long readVarintAt(long p, long limit) {
         long value = 0;
         int shift = 0;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
index f74c8655..27523de4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
@@ -364,8 +364,9 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                 throw new MmapSegmentException(
                         "bad baseSeq in " + path + ": " + baseSeq);
             }
-            long lastGood = scanFrames(addr, fileSize);
-            long count = countFrames(addr, lastGood);
+            FrameScan scan = scanFrames(addr, fileSize);
+            long lastGood = scan.lastGood;
+            long count = scan.frameCount;
             long tornTail = detectTornTail(addr, lastGood, fileSize);
             if (tornTail > 0) {
                 LOG.warn("SF segment {}: torn tail of {} bytes at offset {} "
@@ -907,14 +908,15 @@ private static boolean isMmapAccessFault(Throwable t) {
     }
 
     /**
-     * Forward scan that returns the offset just past the last frame whose
-     * CRC verifies. A torn-tail frame (declared length runs past EOF, or
-     * CRC mismatch) leaves both cursors at the start of that frame; the
-     * next {@link #tryAppend} will overwrite it. The scan only reads from
-     * the mapping — no syscalls.
+     * Forward scan that returns the offset just past the last frame whose CRC
+     * verifies and the number of verified frames. A torn-tail frame (declared
+     * length runs past EOF, or CRC mismatch) leaves both cursors at the start of
+     * that frame; the next {@link #tryAppend} will overwrite it. The scan only
+     * reads from the mapping — no syscalls.
      */
-    private static long scanFrames(long addr, long fileSize) {
+    private static FrameScan scanFrames(long addr, long fileSize) {
         long pos = HEADER_SIZE;
+        long frameCount = 0L;
         try {
             while (pos + FRAME_HEADER_SIZE <= fileSize) {
                 int crcRead = Unsafe.getUnsafe().getInt(addr + pos);
@@ -922,7 +924,7 @@ private static long scanFrames(long addr, long fileSize) {
                 // Defensive: a corrupt length field could be enormous or negative,
                 // both of which would otherwise overrun the mapping.
                 if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 // CRC over the contiguous (payloadLen, payload) pair, folded
                 // via Unsafe reads rather than the native Crc32c.update.
@@ -945,9 +947,10 @@ private static long scanFrames(long addr, long fileSize) {
                 // table CRC here is immaterial.
                 int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen);
                 if (crcCalc != crcRead) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 pos += FRAME_HEADER_SIZE + payloadLen;
+                frameCount++;
             }
         } catch (InternalError e) {
             // The read at `pos` hit a mapped page that is not backed by real
@@ -973,7 +976,7 @@ private static long scanFrames(long addr, long fileSize) {
                             + "if this recurs.",
                     pos, fileSize);
         }
-        return pos;
+        return new FrameScan(pos, frameCount);
     }
 
     /**
@@ -1049,19 +1052,13 @@ private static long detectTornTail(long addr, long lastGood, long fileSize) {
         return 0L;
     }
 
-    /**
-     * Counts frames in {@code [HEADER_SIZE, lastGood)}. Walks the framing in
-     * lockstep with {@link #scanFrames} (which already validated CRCs); so
-     * this is just length-driven traversal, no CRC re-check.
-     */
-    private static long countFrames(long addr, long lastGood) {
-        long pos = HEADER_SIZE;
-        long count = 0;
-        while (pos < lastGood) {
-            int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4);
-            pos += FRAME_HEADER_SIZE + payloadLen;
-            count++;
+    private static final class FrameScan {
+        private final long frameCount;
+        private final long lastGood;
+
+        private FrameScan(long lastGood, long frameCount) {
+            this.lastGood = lastGood;
+            this.frameCount = frameCount;
         }
-        return count;
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
index 982f93c2..914e4ab5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
@@ -163,6 +163,7 @@ public final class PersistedSymbolDict implements QuietCloseable {
      */
     static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8;
     static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk
+    private static final int[] CRC32C_TABLE = buildCrc32cTable();
     private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class);
     private final int fd;
     // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files);
@@ -172,6 +173,10 @@ public final class PersistedSymbolDict implements QuietCloseable {
     // Production writes directly into segmented append mappings. Custom facades retain the
     // positioned-write path so fault tests can inject short writes through ff.write.
     private final boolean mappedAppend;
+    // True only when recovery parsed the file through a temporary read-only
+    // mmap instead of allocating a second native buffer as large as the file.
+    // Test-visible so the peak-memory regression has an observable contract.
+    private final boolean mappedRecoveryInput;
     private long appendMapAddr;
     private long appendMapCapacity;
     private long appendMapOffset;
@@ -182,21 +187,29 @@ public final class PersistedSymbolDict implements QuietCloseable {
     // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and
     // CRCs stripped -- populated only when open() recovered existing chunks
     // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not
-    // consumed) to seed the producer's id map (readLoadedSymbols) and to seed the
-    // send loop's catch-up mirror, which COPIES it. This file retains ownership for
-    // the engine's lifetime -- the orphan drainer builds a fresh send loop per wire
-    // session against the same engine, and each must re-seed its mirror -- and frees
-    // this buffer in close().
+    // consumed) to seed the producer's id map and to seed the send loop's catch-up
+    // mirror. Foreground construction transfers ownership to its sole loop after
+    // producer seeding; orphan-drainer loops borrow it because one engine may create
+    // several wire sessions. If ownership was not transferred, close() frees it.
     private long loadedEntriesAddr;
     private int loadedEntriesLen;
     private long scratchAddr;
     private int scratchCap;
     private int size;
 
-    private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) {
+    private PersistedSymbolDict(
+            FilesFacade ff,
+            int fd,
+            long appendOffset,
+            int size,
+            long loadedEntriesAddr,
+            int loadedEntriesLen,
+            boolean mappedRecoveryInput
+    ) {
         this.ff = ff;
         this.fd = fd;
         this.mappedAppend = ff == FilesFacade.INSTANCE;
+        this.mappedRecoveryInput = mappedRecoveryInput;
         this.appendOffset = appendOffset;
         this.size = size;
         this.loadedEntriesAddr = loadedEntriesAddr;
@@ -243,13 +256,9 @@ public static PersistedSymbolDict open(FilesFacade ff, String slotDir) {
             return null;
         }
         if (existing >= HEADER_SIZE) {
-            // A dictionary at or past Integer.MAX_VALUE cannot be read back:
-            // openExisting reads it into ONE int-sized native buffer, and the (int)
-            // cast is either negative (malloc rejects it) or a small positive prefix
-            // (whose parse would then trim the real multi-GB file). Degrade to full
-            // self-sufficient frames and leave the file alone. Reaching this needs
-            // ~100M+ distinct symbols on one slot, far past realistic cardinality;
-            // the guard keeps the read/write size boundary safe anyway.
+            // Chunk lengths and the retained contiguous entry region are int-sized,
+            // so a dictionary at or past Integer.MAX_VALUE cannot be represented
+            // safely even though production recovery maps rather than reads it.
             if (existing >= Integer.MAX_VALUE) {
                 LOG.warn("symbol dict {} too large ({} bytes) to reopen; "
                         + "falling back to full-dictionary frames (file left intact)", filePath, existing);
@@ -541,17 +550,45 @@ public int loadedEntriesLen() {
     }
 
     /**
-     * Materialises the loaded entries as symbol strings in ascending-id order
-     * (entry {@code i} is symbol id {@code i}). Used once on recovery to
-     * repopulate the producer's global dictionary. Empty when nothing was
-     * recovered.
-     * 
      
-     * Construction-phase only -- like {@link #loadedEntriesAddr()}, this
-     * walks the native entry region {@link #close()} frees, with no closed-guard,
-     * so it must run before the I/O thread and any producer append start.
+     * Transfers the recovered entry buffer to the foreground send loop. The
+     * symbol count remains unchanged because it is also the append baseline;
+     * only the native-buffer ownership moves. Construction-phase only.
+     */
+    synchronized long takeLoadedEntries() {
+        long addr = loadedEntriesAddr;
+        loadedEntriesAddr = 0L;
+        loadedEntriesLen = 0;
+        return addr;
+    }
+
+    @TestOnly
+    public boolean usedMappedRecoveryInput() {
+        return mappedRecoveryInput;
+    }
+
+    /**
+     * Decodes the recovered entries directly into {@code target} in ascending-id
+     * order. This avoids materialising a cardinality-sized temporary list that
+     * the producer would immediately copy into the global dictionary.
+     * Construction-phase only; see {@link #loadedEntriesAddr()}.
      */
+    public void addLoadedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeLoadedSymbols(target, null);
+    }
+
+    /**
+     * Materialises the loaded entries as symbol strings in ascending-id order.
+     * Retained for recovery-format tests; production decodes directly through
+     * {@link #addLoadedSymbolsTo(GlobalSymbolDictionary)}.
+     */
+    @TestOnly
     public ObjList readLoadedSymbols() {
         ObjList out = new ObjList<>(Math.max(size, 1));
+        decodeLoadedSymbols(null, out);
+        return out;
+    }
+
+    private void decodeLoadedSymbols(GlobalSymbolDictionary target, ObjList out) {
         long p = loadedEntriesAddr;
         long limit = p + loadedEntriesLen;
         for (int i = 0; i < size && p < limit; i++) {
@@ -575,10 +612,14 @@ public ObjList readLoadedSymbols() {
             if (p + len > limit) {
                 break; // defensive: torn tail (should not happen past parse in open)
             }
-            out.add(Utf8s.stringFromUtf8Bytes(p, p + len));
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + len);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
             p += len;
         }
-        return out;
     }
 
     /**
@@ -595,91 +636,48 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath,
                     + "falling back to full-dictionary frames (file left intact)", filePath, fd);
             return null;
         }
-        long buf = 0L;
+        int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
+        boolean mappedInput = ff == FilesFacade.INSTANCE;
+        long inputAddr = 0L;
         long entriesAddr = 0L;
-        int entriesCap = 0;
+        int entriesLen = 0;
         try {
-            int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
-            buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
-            long read = ff.read(fd, buf, len, 0);
-            if (read != len
-                    || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC
-                    || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) {
-                LOG.warn("symbol dict {} unreadable, bad magic or unknown version; "
-                        + "falling back to full-dictionary frames (file left intact)", filePath);
-                Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT);
-                buf = 0L; // null after free so the catch below cannot double-free if ff.close throws
-                int fdToClose = fd;
-                fd = -1; // relinquish before close so the catch cannot double-close if close throws
-                ff.close(fdToClose);
-                return null;
-            }
-            // Parse the chunks after the header, copying each chunk's entry region
-            // into entriesAddr AS WE VALIDATE IT -- one pass over the file, not two.
-            // Every chunk sheds its two header varints and its CRC, so the entry
-            // region is strictly smaller than the file and len - HEADER_SIZE is a safe
-            // upper bound to allocate up front; we shrink to the exact size below.
-            //
-            // Stop at the first torn/incomplete OR crc-mismatched chunk. The CRC turns
-            // an interior tear (a lost page reading back as zeroes) or a stale chunk
-            // left past the end by a failed truncate into a clean stop point, so
-            // recovery trusts only the intact prefix instead of silently mis-parsing a
-            // corrupt chunk and shifting the dense id->symbol map.
-            entriesCap = len - HEADER_SIZE;
-            long dst = 0L;
-            if (entriesCap > 0) {
-                entriesAddr = Unsafe.malloc(entriesCap, MemoryTag.NATIVE_DEFAULT);
-                dst = entriesAddr;
-            }
-            Varint v = new Varint();
-            int diskPos = HEADER_SIZE;
-            int count = 0;
-            while (diskPos < len) {
-                if (!v.decode(buf, diskPos, len)) {
-                    break; // torn entryCount varint
-                }
-                long entryCount = v.value;
-                if (!v.decode(buf, v.end, len)) {
-                    break; // torn entryBytes varint
-                }
-                long entryBytes = v.value;
-                int entriesStart = v.end;
-                // entryCount/entryBytes stay long so a corrupt multi-gigabyte value
-                // cannot wrap an int back under the bound checks.
-                long chunkEnd = (long) entriesStart + entryBytes; // end of the entry region
-                if (chunkEnd + CRC_SIZE > len) {
-                    break; // torn/incomplete trailing chunk (its CRC doesn't fit)
+            if (mappedInput) {
+                inputAddr = Files.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT);
+                if (inputAddr == Files.FAILED_MMAP_ADDRESS) {
+                    inputAddr = 0L;
+                    throw new IllegalStateException("could not mmap symbol dictionary for recovery");
                 }
-                int chunkEndI = (int) chunkEnd;
-                int crcStored = Unsafe.getUnsafe().getInt(buf + chunkEndI);
-                int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, chunkEndI - diskPos);
-                if (crcCalc != crcStored) {
-                    break; // corrupt/stale chunk -- stop before it (fail-clean)
+            } else {
+                // Fault-injection facades retain the positioned-read path so tests
+                // can still force short reads without mapping around the seam.
+                inputAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+                long read = ff.read(fd, inputAddr, len, 0);
+                if (read != len) {
+                    throw new IllegalStateException("short read while recovering symbol dictionary "
+                            + "[expected=" + len + ", actual=" + read + ']');
                 }
-                // A chunk carries at least one entry, and the ids are int-dense. A
-                // CRC-valid chunk cannot violate either at realistic cardinality, but
-                // keep the int narrowing honest rather than wrapping the id space.
-                if (entryCount <= 0 || (long) count + entryCount > Integer.MAX_VALUE) {
-                    break;
-                }
-                Unsafe.getUnsafe().copyMemory(buf + entriesStart, dst, entryBytes);
-                dst += entryBytes;
-                diskPos = chunkEndI + CRC_SIZE;
-                count += (int) entryCount;
             }
-            int entriesLen = entriesAddr != 0L ? (int) (dst - entriesAddr) : 0;
-            int diskConsumed = diskPos - HEADER_SIZE; // valid chunks incl. headers and CRCs
-            if (entriesAddr != 0L && entriesLen == 0) {
-                Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT);
-                entriesAddr = 0L;
-                entriesCap = 0;
-            } else if (entriesAddr != 0L && entriesLen < entriesCap) {
-                // Shrink the upper-bound allocation to what the trusted prefix used.
-                entriesAddr = Unsafe.realloc(entriesAddr, entriesCap, entriesLen, MemoryTag.NATIVE_DEFAULT);
-                entriesCap = entriesLen;
+            if (Unsafe.getUnsafe().getInt(inputAddr) != FILE_MAGIC
+                    || Unsafe.getUnsafe().getByte(inputAddr + 4) != VERSION) {
+                throw new IllegalStateException("bad magic or unknown symbol dictionary version");
+            }
+            // First pass validates chunks and totals only the entry bytes that
+            // survive. Production reads through a file-backed mapping, so this
+            // pass does not allocate an anonymous whole-file copy. The second
+            // pass copies directly into one exact-sized retained allocation.
+            RecoveryScan scan = scanRecoveredChunks(inputAddr, len);
+            entriesLen = scan.entriesLen;
+            if (entriesLen > 0) {
+                entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT);
+                copyRecoveredEntries(inputAddr, scan.validLen, entriesAddr, entriesLen);
             }
-            Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT);
-            buf = 0L;
+            if (mappedInput) {
+                Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+            } else {
+                Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+            inputAddr = 0L;
             // Drop any torn/stale trailing bytes so a LATER, shorter append cannot
             // leave residue past its own end. The truncate result IS checked: a file
             // we cannot trim could still expose stale post-end bytes whose
@@ -687,36 +685,23 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath,
             // position, so a failed truncate makes the file untrusted -- return null
             // (the sender falls back to full self-sufficient frames) and, per the
             // never-destroy contract, leave every byte on disk.
-            long validLen = HEADER_SIZE + diskConsumed;
-            if (validLen < len && !ff.truncate(fd, validLen)) {
-                LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); "
-                        + "falling back to full-dictionary frames (file left intact)", filePath);
-                if (entriesAddr != 0L) {
-                    Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT);
-                    // Null after freeing (like buf above) so the catch below cannot
-                    // double-free entriesAddr if the following ff.close throws.
-                    entriesAddr = 0L;
-                    entriesCap = 0;
-                }
-                int fdToClose = fd;
-                fd = -1; // relinquish before close so the catch cannot double-close if close throws
-                ff.close(fdToClose);
-                return null;
+            if (scan.validLen < len && !ff.truncate(fd, scan.validLen)) {
+                throw new IllegalStateException("could not drop torn/stale symbol dictionary tail");
             }
-            return new PersistedSymbolDict(ff, fd, validLen, count, entriesAddr, entriesLen);
+            return new PersistedSymbolDict(
+                    ff, fd, scan.validLen, scan.count, entriesAddr, entriesLen, mappedInput);
         } catch (Throwable t) {
-            if (buf != 0L) {
-                Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT);
+            if (inputAddr != 0L) {
+                if (mappedInput) {
+                    Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+                } else {
+                    Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+                }
             }
-            // Free entriesAddr if it was allocated and not yet handed off. The success
-            // path transfers it to the returned dict, and every path that frees it
-            // earlier (the truncate-failure branch above) also nulls it, so this cannot
-            // double-free. Keeps the error path leak-free on any throw between its
-            // malloc and the return.
             if (entriesAddr != 0L) {
-                Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT);
+                Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT);
             }
-            if (fd >= 0) { // a branch that already closed fd relinquished it to -1
+            if (fd >= 0) {
                 ff.close(fd);
             }
             // Pass the throwable as a trailing argument with no matching placeholder so
@@ -728,6 +713,79 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath,
         }
     }
 
+    private static void copyRecoveredEntries(long inputAddr, int validLen, long entriesAddr, int entriesLen) {
+        Varint v = new Varint();
+        int diskPos = HEADER_SIZE;
+        int dstPos = 0;
+        while (diskPos < validLen) {
+            if (!v.decode(inputAddr, diskPos, validLen)
+                    || !v.decode(inputAddr, v.end, validLen)) {
+                throw new IllegalStateException("validated symbol dictionary chunk could not be decoded");
+            }
+            int entryBytes = (int) v.value;
+            int entriesStart = v.end;
+            Unsafe.getUnsafe().copyMemory(inputAddr + entriesStart, entriesAddr + dstPos, entryBytes);
+            dstPos += entryBytes;
+            diskPos = entriesStart + entryBytes + CRC_SIZE;
+        }
+        assert dstPos == entriesLen;
+    }
+
+    private static int[] buildCrc32cTable() {
+        int[] table = new int[256];
+        for (int n = 0; n < 256; n++) {
+            int c = n;
+            for (int k = 0; k < 8; k++) {
+                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
+            }
+            table[n] = c;
+        }
+        return table;
+    }
+
+    private static int crc32cRecovery(long addr, long len) {
+        int crc = ~Crc32c.INIT;
+        for (long i = 0; i < len; i++) {
+            crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF];
+        }
+        return ~crc;
+    }
+
+    private static RecoveryScan scanRecoveredChunks(long inputAddr, int len) {
+        Varint v = new Varint();
+        int count = 0;
+        int diskPos = HEADER_SIZE;
+        long entriesLen = 0L;
+        while (diskPos < len) {
+            if (!v.decode(inputAddr, diskPos, len)) {
+                break;
+            }
+            long entryCount = v.value;
+            if (!v.decode(inputAddr, v.end, len)) {
+                break;
+            }
+            long entryBytes = v.value;
+            int entriesStart = v.end;
+            long chunkEnd = (long) entriesStart + entryBytes;
+            if (chunkEnd + CRC_SIZE > len) {
+                break;
+            }
+            int chunkEndI = (int) chunkEnd;
+            int crcStored = Unsafe.getUnsafe().getInt(inputAddr + chunkEndI);
+            int crcCalc = crc32cRecovery(inputAddr + diskPos, chunkEndI - diskPos);
+            if (crcCalc != crcStored
+                    || entryCount <= 0
+                    || (long) count + entryCount > Integer.MAX_VALUE
+                    || entriesLen + entryBytes > Integer.MAX_VALUE) {
+                break;
+            }
+            entriesLen += entryBytes;
+            diskPos = chunkEndI + CRC_SIZE;
+            count += (int) entryCount;
+        }
+        return new RecoveryScan(count, (int) entriesLen, diskPos);
+    }
+
     private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
         int fd = ff.openCleanRW(filePath);
         if (fd < 0) {
@@ -770,7 +828,7 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
                 Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
             }
         }
-        return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0);
+        return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0, false);
     }
 
     /**
@@ -962,11 +1020,22 @@ private void writeChunkFromScratch(int entriesLen, int count) {
         flushChunk(recStart, hdrLen, entriesLen, count);
     }
 
+    private static final class RecoveryScan {
+        private final int count;
+        private final int entriesLen;
+        private final int validLen;
+
+        private RecoveryScan(int count, int entriesLen, int validLen) {
+            this.count = count;
+            this.entriesLen = entriesLen;
+            this.validLen = validLen;
+        }
+    }
+
     /**
-     * Zero-allocation LEB128 decoder, one instance per {@link #openExisting} call --
-     * not one per chunk. The previous {@code long[]}-returning decoder allocated once
-     * per ENTRY, so a million-symbol recovery churned a million throwaway arrays in a
-     * class that is otherwise strictly allocation-free.
+     * Zero-allocation LEB128 decoder, one instance per recovery pass -- not one
+     * per chunk. The previous {@code long[]}-returning decoder allocated once per
+     * entry, so a million-symbol recovery churned a million throwaway arrays.
      */
     private static final class Varint {
         int end;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
index 386d0f1e..e94e68f5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.client.sf.cursor;
 
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.ObjList;
@@ -55,6 +56,7 @@ final class RecoveredFrameAnalysis implements QuietCloseable {
     private boolean runningUnackedGap;
     private long runningMaxDeltaEnd;
     private long runningMaxDeltaStart;
+    private long symbolEntriesVisited;
     private long rawAddr;
     private int rawCapacity;
     private int runningRawCount;
@@ -95,6 +97,14 @@ void accept(long fsn, long payload, int payloadLen) {
     }
 
     void appendDecodedSymbols(ObjList out) {
+        decodeSymbols(null, out);
+    }
+
+    void addDecodedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeSymbols(target, null);
+    }
+
+    private void decodeSymbols(GlobalSymbolDictionary target, ObjList out) {
         long p = rawAddr;
         long limit = rawAddr + committedRawLen;
         for (int i = 0; i < committedRawCount; i++) {
@@ -108,7 +118,12 @@ void appendDecodedSymbols(ObjList out) {
             if (symbolLen > limit - p) {
                 throw new IllegalStateException("truncated cached symbol dictionary suffix");
             }
-            out.add(Utf8s.stringFromUtf8Bytes(p, p + symbolLen));
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + symbolLen);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
             p += symbolLen;
         }
         if (p != limit) {
@@ -152,6 +167,10 @@ int rawLen() {
         return committedRawLen;
     }
 
+    long symbolEntriesVisited() {
+        return symbolEntriesVisited;
+    }
+
     @Override
     public void close() {
         if (rawAddr != 0L) {
@@ -227,9 +246,16 @@ private void foldDelta(long fsn, long p, long limit) {
             markGap(fsn);
             return;
         }
+        // The segment scan already CRC-validated this frame. When its entire
+        // range is covered, entry parsing cannot extend recovery state, so skip
+        // the cardinality-proportional payload walk.
+        if (deltaEnd <= runningCoverage) {
+            return;
+        }
 
         long id = deltaStart;
         for (long i = 0; i < deltaCount; i++, id++) {
+            symbolEntriesVisited++;
             long entryStart = p;
             long encodedLen = readVarint(p, limit);
             if (encodedLen < 0L) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
index acc8102a..86ba58a0 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
@@ -960,6 +960,7 @@ private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols)
             }
         }
         setField(loop, "sentDictBytesAddr", addr);
+        setBooleanField(loop, "sentDictBytesOwned", true);
         setIntField(loop, "sentDictBytesCapacity", total);
         setIntField(loop, "sentDictBytesLen", total);
         setIntField(loop, "sentDictCount", symbols.length);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
index 8697eaee..116cbc55 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
@@ -81,6 +81,7 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception {
                     Assert.assertNotNull("disk-mode engine must open a persisted dict", pd);
                     Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)",
                             pd.size() > 0 && pd.loadedEntriesLen() > 0);
+                    long persistedAddr = pd.loadedEntriesAddr();
 
                     CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
                             null, engine, 0, 1_000_000L,
@@ -92,6 +93,12 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception {
                     // thread's to free, since the I/O loop never ran.
                     Assert.assertTrue("precondition: the ctor seeded a non-empty mirror",
                             readInt(loop, "sentDictCount") > 0);
+                    Assert.assertEquals("foreground loop must take the persisted buffer without copying",
+                            persistedAddr, readLong(loop, "sentDictBytesAddr"));
+                    Assert.assertEquals("ownership transfer must clear the persisted pointer",
+                            0L, pd.loadedEntriesAddr());
+                    Assert.assertTrue("foreground mirror must own the transferred buffer",
+                            readBoolean(loop, "sentDictBytesOwned"));
                     loop.close();
                     // close() must reset sentDictCount alongside freeing the buffer,
                     // so the mirror stays all-or-nothing: a hypothetical post-close
@@ -111,14 +118,14 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
         // C1 regression: the orphan drainer (BackgroundDrainer) builds a NEW
         // CursorWebSocketSendLoop per wire session against the SAME, persistent
         // engine when a durable-ack capability gap forces a mid-drain recycle. The
-        // recovery mirror seed must survive that recycle. If the first loop CONSUMED
+        // recovery mirror seed must survive that recycle. If the first loop consumes
         // the persisted dictionary's loaded entries (a one-shot ownership transfer),
         // the second loop seeds an EMPTY mirror (sentDictCount = 0), sends no
         // reconnect catch-up, and the first replayed delta frame (deltaStart > 0)
         // trips the torn-dict guard -- falsely quarantining a healthy slot with a
-        // bogus "resend required" terminal. Copying the entries (leaving the
-        // dictionary intact for the engine's lifetime) lets every recycled loop
-        // re-seed. Pre-fix, loop2's sentDictCount is 0 and this assertion fails.
+        // bogus "resend required" terminal. Borrowing the entries leaves the
+        // dictionary intact for the engine's lifetime without making another native
+        // copy, so every recycled loop can re-seed.
         Path sfDir = Files.createTempDirectory("qwp-mirror-reseed");
         try {
             populateRecoverableSlot(sfDir);
@@ -129,15 +136,22 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
                     Assert.assertNotNull(pd);
                     int dictSize = pd.size();
                     Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0);
+                    long persistedAddr = pd.loadedEntriesAddr();
 
                     // Session 1 seeds its mirror from the persisted dictionary.
                     CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine);
                     try {
                         Assert.assertEquals("session-1 mirror must seed from the persisted dict",
                                 dictSize, readInt(loop1, "sentDictCount"));
+                        Assert.assertEquals("orphan session must borrow the persisted bytes",
+                                persistedAddr, readLong(loop1, "sentDictBytesAddr"));
+                        Assert.assertFalse("borrowed orphan mirror must not own the persisted bytes",
+                                readBoolean(loop1, "sentDictBytesOwned"));
                     } finally {
                         loop1.close();
                     }
+                    Assert.assertEquals("closing a borrowed loop must leave the engine prefix alive",
+                            persistedAddr, pd.loadedEntriesAddr());
 
                     // Session 2 against the SAME engine (the drainer recycle): the
                     // seed must NOT have been consumed -- the mirror must re-seed to
@@ -147,6 +161,7 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
                         Assert.assertEquals("recycled session-2 mirror must re-seed from the "
                                         + "persisted dict (pre-fix it was 0)",
                                 dictSize, readInt(loop2, "sentDictCount"));
+                        Assert.assertEquals(persistedAddr, readLong(loop2, "sentDictBytesAddr"));
                     } finally {
                         loop2.close();
                     }
@@ -205,6 +220,8 @@ public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception {
                     } finally {
                         setMirrorSeedFault(false);
                     }
+                    Assert.assertTrue("failed foreground construction must leave the persisted prefix owned",
+                            pd.loadedEntriesAddr() != 0L);
                     // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor
                     // malloc'd was freed on the throw -- pre-fix it leaks here.
                 }
@@ -223,7 +240,9 @@ private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine)
                 () -> {
                     throw new IOException("no reconnect in this test");
                 },
-                0, 0, 1);
+                0, 0, 1,
+                false, 0L, 3, 0L, 0L,
+                CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET);
     }
 
     private static void populateRecoverableSlot(Path sfDir) throws Exception {
@@ -275,6 +294,18 @@ private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exc
         return f.getInt(loop);
     }
 
+    private static boolean readBoolean(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getBoolean(loop);
+    }
+
+    private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getLong(loop);
+    }
+
     // Toggles the loop's @TestOnly mirror-seed fault flag. Reflection because the
     // flag is package-private in the production package (this test is in a sibling
     // test package), the same non-reflective-path-unavailable reason readInt uses.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
index c96bfefe..d7010310 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
@@ -27,10 +27,12 @@
 import io.questdb.client.DefaultHttpClientConfiguration;
 import io.questdb.client.cutlass.http.client.WebSocketClient;
 import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
 import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
 import io.questdb.client.network.PlainSocketFactory;
 import io.questdb.client.std.Files;
 import io.questdb.client.std.MemoryTag;
@@ -399,6 +401,35 @@ public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exce
         });
     }
 
+    @Test
+    public void testRecoverySkipsEntriesAlreadyCoveredByPersistedPrefix() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                assertNotNull(pd);
+                pd.appendSymbol("a");
+                pd.appendSymbol("b");
+                pd.appendSymbol("c");
+                appendDeltaSymbolFrame(engine, 0, 'a');
+                appendDeltaSymbolFrame(engine, 1, 'b');
+                appendDeltaSymbolFrame(engine, 2, 'c');
+            }
+
+            try (CursorSendEngine engine = newEngine()) {
+                assertEquals("covered delta payloads must not be parsed entry-by-entry",
+                        0L, engine.recoverySymbolEntriesVisited());
+                GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+                PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                assertNotNull(pd);
+                pd.addLoadedSymbolsTo(recovered);
+                assertEquals(3L, engine.addRecoveredSymbolsTo(recovered.size(), recovered));
+                assertEquals("direct decode must not duplicate the covered frame entries",
+                        3, recovered.size());
+                assertEquals("c", recovered.getSymbol(2));
+            }
+        });
+    }
+
     @Test
     public void testSelfSufficientFrameRepairsAckedRecoveryGap() throws Exception {
         // fsn 0 models the tail of an old delta epoch whose registering frames
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
index 4707c233..da9c4494 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
@@ -100,6 +100,8 @@ public void testCreateAppendCloseReopenScansAllFrames() throws Exception {
                     long expectedEnd = MmapSegment.HEADER_SIZE
                             + 100L * (MmapSegment.FRAME_HEADER_SIZE + 32);
                     assertEquals(expectedEnd, seg.publishedOffset());
+                    assertEquals("validation scan must return the frame count",
+                            100L, seg.frameCount());
                 }
 
                 // Re-open: scan must land at exactly the same offset.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
index aea6d1af..ee046d33 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
@@ -77,6 +77,13 @@ public void testAppendPersistsAcrossReopen() throws Exception {
                 Assert.assertEquals("GOOG", symbols.getQuick(1));
                 Assert.assertEquals("MSFT", symbols.getQuick(2));
                 Assert.assertTrue(reopened.loadedEntriesLen() > 0);
+                Assert.assertTrue("production recovery must parse through mmap instead of a heap file copy",
+                        reopened.usedMappedRecoveryInput());
+
+                GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+                reopened.addLoadedSymbolsTo(recovered);
+                Assert.assertEquals("direct recovery decode must preserve dense ids", 3, recovered.size());
+                Assert.assertEquals("GOOG", recovered.getSymbol(1));
 
                 // Appending after recovery continues from the recovered tip.
                 reopened.appendSymbol("TSLA");

From 5e467f3763dc1fc9a139bd735419ff780757728f Mon Sep 17 00:00:00 2001
From: glasstiger 
Date: Thu, 16 Jul 2026 15:52:07 +0100
Subject: [PATCH 78/78] fix: harden reconnect and test cleanup handling

---
 .../qwp/client/QwpWebSocketSender.java        |   2 +-
 .../client/sf/cursor/BackgroundDrainer.java   |  11 +-
 .../sf/cursor/CursorWebSocketSendLoop.java    | 220 +++++++++------
 .../cutlass/qwp/client/ReconnectTest.java     |  30 +--
 .../BackgroundDrainerDurableAckRetryTest.java |   4 +-
 ...SendLoopForegroundReconnectPolicyTest.java | 254 ++++++++++++++++++
 .../questdb/client/test/tools/TestUtils.java  |  85 +++++-
 .../client/test/tools/TestUtilsTest.java      |  65 +++++
 8 files changed, 551 insertions(+), 120 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java
 create mode 100644 core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 1fba1452..557af871 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -3430,7 +3430,7 @@ private void ensureConnected() {
                     maxFrameRejections,
                     poisonMinEscalationWindowMillis,
                     catchUpCapGapMinEscalationWindowMillis,
-                    CursorWebSocketSendLoop.CatchUpCapGapPolicy.RETRY_FOREVER);
+                    CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
             // Plug the async-delivery sink before start() so the I/O thread
             // never observes a null dispatcher between recordFatal and
             // notification — the test for null in dispatchError handles
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
index 38b6b036..03ebee3e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
@@ -243,8 +243,10 @@ public BackgroundDrainer() {
      * durable ack -- i.e. the symptom of a misconfigured cluster or a
      * rolling-upgrade transient.
      * 
      
-     * For the foreground sender that condition is loud-fail: the producer
-     * is actively pushing data. The drainer is asymmetric: source data is
+     * For a foreground sender's initial connection that condition is loud-fail;
+     * after the sender has been live, its reconnect loop keeps buffering and
+     * retrying through a rolling capability change. The drainer is asymmetric:
+     * source data is
      * pinned (durable-ack-mode trims only on STATUS_DURABLE_ACK frames,
      * which the offending endpoints by definition do not send), so we
      * give the cluster a budget to settle before quarantining the slot.
@@ -318,8 +320,7 @@ public WebSocketClient connectWithDurableAckRetry() {
             } catch (QwpAuthFailedException | WebSocketUpgradeException e) {
                 // Genuinely non-retriable across the cluster (auth 401/403, or a
                 // non-421 upgrade reject): waiting will not fix it, so quarantine
-                // immediately -- exactly as the live sender's background loop
-                // (CursorWebSocketSendLoop.connectLoop) halts on these errors.
+                // immediately under the orphan reconnect policy.
                 String msg = e.getMessage();
                 LOG.error("drainer terminal upgrade/auth error for slot {}: {}", slotPath, msg);
                 lastErrorMessage = msg;
@@ -639,7 +640,7 @@ public void run() {
                         maxHeadFrameRejections,
                         poisonMinEscalationWindowMillis,
                         catchUpCapGapMinEscalationWindowMillis,
-                        CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET);
+                        CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
                 loop.start();
 
                 while (!stopRequestedOrInterrupted()) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 3d33bfd0..e9af2d5d 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -70,8 +70,10 @@
  *   
    6. On wire failure, runs the configured reconnect policy: capped
  *       exponential backoff with jitter, retried indefinitely (Invariant B --
  *       a store-and-forward drainer never gives up on a wall-clock budget),
- *       with only auth-style failures (401/403/non-101 upgrade reject) treated
- *       as terminal. On reconnect success, repositions the cursor at
+ *       with endpoint-policy failures terminal only for an initial foreground
+ *       connect or an orphan drainer. A previously-live foreground sender keeps
+ *       retrying so credential and rolling-capability changes remain contained
+ *       by store-and-forward. On reconnect success, repositions the cursor at
  *       {@code ackedFsn+1} and replays.
      7. 
  *
    * No locks on the steady-state path. The producer thread (user) writes @@ -225,6 +227,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // user-facing 5-minute default is applied at the config layer. private final long catchUpCapGapMinEscalationWindowNanos; private final CatchUpCapGapPolicy catchUpCapGapPolicy; + private final ReconnectPolicy reconnectPolicy; private final CursorSendEngine engine; private final long parkNanos; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only @@ -370,9 +373,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a // successful connect attempt in any mode). Once true, stays true. Used to - // distinguish a "never reached the server" terminal failure (looks like a - // config typo or firewall block) from "lost connection after we were - // up" (looks transient). + // distinguish an initial endpoint-policy failure (fail fast) from a + // post-start failure that a foreground sender must contain and retry. private volatile boolean hasEverConnected; private volatile Thread ioThread; // Typed marker for a durable-ack CAPABILITY-GAP terminal: set (before the @@ -381,8 +383,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // QwpDurableAckMismatchException. The orphan drainer consults it to route // a mid-drain capability gap into its budgeted settle-retry // (BackgroundDrainer.connectWithDurableAckRetry) instead of quarantining - // the slot on the first sweep; the foreground sender ignores it and keeps - // its spec'd loud-fail (sf-client.md section 8.1). Write-once alongside + // the slot on the first sweep. Foreground reconnects never set this marker; + // they keep retrying after a successful initial connection. Write-once alongside // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; @@ -512,9 +514,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code client} may be {@code null} only if {@code reconnectFactory} * is non-null — this is the async-initial-connect path: the I/O thread * runs the same retry loop on its first iteration to obtain a live - * client, and a terminal failure (auth/upgrade reject) is delivered - * through the dispatcher rather than thrown to the constructor's - * caller; plain connect failures are retried indefinitely + * client, and an initial terminal failure (auth/upgrade reject or durable-ack + * mismatch) is delivered through the dispatcher rather than thrown to the + * constructor's caller; plain connect failures are retried indefinitely * (Invariant B: no wall-clock budget give-up). */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, @@ -656,10 +658,9 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } /** - * Policy-aware master constructor. Foreground senders must pass - * {@link CatchUpCapGapPolicy#RETRY_FOREVER}; only orphan drainers may pass - * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET} and quarantine a slot - * after the attempt and dwell thresholds are both exhausted. + * Compatibility policy-aware constructor. New production call sites should + * use the {@link ReconnectPolicy} overload below, which names the foreground + * versus orphan ownership distinction directly. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -696,6 +697,9 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); } this.catchUpCapGapPolicy = catchUpCapGapPolicy; + this.reconnectPolicy = catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER + ? ReconnectPolicy.FOREGROUND + : ReconnectPolicy.ORPHAN; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -831,6 +835,41 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } + /** + * Policy-aware master constructor. A foreground sender fails fast while + * establishing its first connection, then retries endpoint-policy failures + * indefinitely after it has been live. An orphan drainer returns such failures + * to its owner so the slot can follow its settle/quarantine policy. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + ReconnectPolicy reconnectPolicy) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + catchUpPolicyFor(reconnectPolicy)); + } + + private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) { + if (reconnectPolicy == null) { + throw new IllegalArgumentException("reconnectPolicy must be non-null"); + } + return reconnectPolicy == ReconnectPolicy.FOREGROUND + ? CatchUpCapGapPolicy.RETRY_FOREVER + : CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET; + } + /** * Maps a server status byte to a {@link SenderError.Category}. Exposed for unit tests. */ @@ -1487,13 +1526,15 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); - // INVARIANT B: a store-and-forward drainer must NEVER terminate on a + // INVARIANT B: a store-and-forward loop must NEVER terminate on a // wall-clock reconnect budget. A replica-only / all-endpoints-replica // window is TRANSIENT -- a replica gets promoted, a primary reappears -- // so this background loop retries for as long as it is running, backing - // off between attempts. The ONLY terminal conditions are a genuinely - // non-retriable upgrade (auth / non-421 upgrade / durable-ack capability - // gap), which return directly below, or the sender being stopped. SF + // off between attempts. Endpoint-policy failures (auth / non-421 + // upgrade / durable-ack capability gap) are terminal only for orphan + // drainers and a foreground sender's first connection. Once a foreground + // sender has connected, those states are retried so a credential or cluster + // capability rotation cannot stop its producer. SF // exhaustion is surfaced to the PRODUCER as append backpressure, never // here. reconnect_max_duration_millis is intentionally NOT consulted: it // bounds only the blocking (non-lazy) initial connect in @@ -1557,65 +1598,78 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // observation. Do not let the time spent retrying it satisfy orphan dwell. resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { - // Terminal across all configured endpoints per spec sf-client.md - // section 13.3: auth (401/403) bypasses reconnect and surfaces as - // SECURITY_ERROR. WebSocketUpgradeException reaching here is always - // non-421: QwpUpgradeFailures.classify upstream converts a - // 421-with-X-QuestDB-Role to QwpIngressRoleRejectedException, and a - // 421 without that header walks the transport-error path in - // buildAndConnect and lands as a LineSenderException, falling into - // the Throwable branch below. - LOG.error("terminal upgrade error during {} -- won't retry: {}", - phase, e.getMessage()); - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.SECURITY_ERROR, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "ws-upgrade-failed: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + if (endpointPolicyFailureIsTerminal()) { + // Orphans return control to their quarantine owner; a + // foreground sender that never connected still fails fast. + // WebSocketUpgradeException reaching here is always non-421: + // role rejects are classified into the transient branch below. + LOG.error("terminal upgrade error during {} -- won't retry: {}", + phase, e.getMessage()); + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.SECURITY_ERROR, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "ws-upgrade-failed: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground auth/upgrade policy rejected the connection; " + + "retrying because this sender was previously live -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpDurableAckMismatchException e) { - // Per spec sf-client.md section 8.1: the client opted into durable - // ack but the cluster cannot honour it. Loud fail at connect rather - // than silently waiting for ack frames that will never arrive. - // Classified as PROTOCOL_VIOLATION (config/capability mismatch), - // not SECURITY_ERROR -- this is not an auth failure. - LOG.error("durable-ack mismatch during {} -- won't retry: {}", - phase, e.getMessage()); - if (terminalError == null) { - // Mirror recordFatal's first-writer-wins latch: only the - // sweep that owns the terminal may mark the gap, and the - // marker must be visible before the terminalError volatile - // write that checkError() keys on. - capabilityGapTerminal = e; + if (endpointPolicyFailureIsTerminal()) { + // Orphans hand a capability gap back to BackgroundDrainer's + // settle budget. An initial foreground connect remains loud. + LOG.error("durable-ack mismatch during {} -- won't retry: {}", + phase, e.getMessage()); + if (terminalError == null) { + // Publish the marker before terminalError, which is the + // volatile first-writer-wins latch observed by the owner. + capabilityGapTerminal = e; + } + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "durable-ack-mismatch: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground durable-ack capability is temporarily " + + "unavailable; retrying because this sender was previously live -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; } - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.PROTOCOL_VIOLATION, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "durable-ack-mismatch: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; } catch (QwpRoleMismatchException | QwpIngressRoleRejectedException e) { // Role mismatch: every reachable endpoint role-rejected the // upgrade -- right now they are all replicas / primary-catchup. @@ -1704,6 +1758,10 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, lastMsg); } + private boolean endpointPolicyFailureIsTerminal() { + return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; + } + /** * Send {@code err} to the async-delivery dispatcher if one is configured. * Producer-side typed throw (TERMINAL) goes through {@code recordFatal} + @@ -2949,13 +3007,21 @@ public enum CatchUpCapGapPolicy { TERMINAL_AFTER_SETTLE_BUDGET } + /** Identifies who owns data while the reconnect loop is unavailable. */ + public enum ReconnectPolicy { + /** A live producer keeps buffering and retries endpoint-policy failures. */ + FOREGROUND, + /** An orphan drainer returns terminal states to its quarantine owner. */ + ORPHAN + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close * the old client (if needed), build a new one with the same auth/TLS * config, connect, perform the WebSocket upgrade, and return it ready - * to send. Throw on a terminal failure (auth rejection, etc.) — the - * I/O loop will treat the throw as fatal. + * to send. The loop's {@link ReconnectPolicy} decides whether endpoint-policy + * failures are retried or returned as terminal. */ @FunctionalInterface public interface ReconnectFactory { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java index 7ba7a1fe..72c8c021 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java @@ -25,7 +25,6 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; import org.junit.Test; @@ -203,14 +202,13 @@ public void testReconnectNeverGivesUpInvariantB() throws Exception { } @Test - public void testTerminalUpgradeErrorAbortsReconnect() throws Exception { + public void testPostStartAuthErrorRemainsBuffered() throws Exception { // Bespoke raw-socket fixture: first connection completes the // WebSocket upgrade and feeds back STATUS_OK ACKs; any subsequent // connection gets HTTP 401 Unauthorized — exercising the - // auth-terminal path. With reconnect_max_duration_millis=10s and - // a 401 happening on the very first reconnect, the cursor I/O - // loop should surface the terminal error within hundreds of ms, - // not after 10s. + // post-start auth-rotation path. Once a foreground sender has been + // live, store-and-forward owns its unacked data: repeated 401s must + // remain contained and retried rather than stopping the producer. try (Auth401AfterFirstConnectionFixture fixture = new Auth401AfterFirstConnectionFixture()) { fixture.start(); @@ -224,9 +222,8 @@ public void testTerminalUpgradeErrorAbortsReconnect() throws Exception { // Wait for first connection to ACK + close waitFor(() -> fixture.acceptedConnections.get() >= 2, 5_000); - long t0 = System.nanoTime(); Throwable observed = null; - long deadline = System.currentTimeMillis() + 5_000; + long deadline = System.currentTimeMillis() + 750; while (System.currentTimeMillis() < deadline) { try { sender.table("foo").longColumn("v", 2L).atNow(); @@ -237,23 +234,10 @@ public void testTerminalUpgradeErrorAbortsReconnect() throws Exception { } Thread.sleep(50); } - long elapsedMs = (System.nanoTime() - t0) / 1_000_000L; - Assert.assertNotNull("expected terminal error after auth rejection", + Assert.assertNull("post-start auth rejection must stay buffered and retriable", observed); - Assert.assertTrue( - "terminal upgrade error must surface well inside the cap; took " - + elapsedMs + "ms (cap was 10000ms)", - elapsedMs < 5_000); - String msg = observed.getMessage() == null ? "" : observed.getMessage(); - Assert.assertTrue( - "error must mention the terminal upgrade failure: " + msg, - msg.contains("WebSocket upgrade failed") - || msg.contains("I/O thread failed") - || msg.contains("401")); - } catch (LineSenderException ignored) { + waitFor(() -> fixture.acceptedConnections.get() >= 3, 5_000); } - // close() rethrows the latched terminal upgrade error - // (commit 052f6ee). Already observed and asserted above. } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java index 0a6b69f8..82885ba7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java @@ -241,8 +241,8 @@ public void testTerminalUpgradeMarksFailedImmediately() throws Exception { CountingListener listener = new CountingListener(); // A genuinely non-retriable upgrade error (non-421 5xx upgrade reject) is // terminal -- waiting will not fix it -- so the drainer quarantines on the - // first attempt, exactly like the live sender's background loop halts on - // auth/upgrade. A TRANSPORT error, by contrast, is transient and is + // first attempt under the orphan reconnect policy. A TRANSPORT error, + // by contrast, is transient and is // retried (see testTransportErrorNeverQuarantinesInvariantB). ScriptedFactory factory = ScriptedFactory.alwaysFailing( () -> new WebSocketUpgradeException(500, null, "server error during upgrade")); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java new file mode 100644 index 00000000..064edf3f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java @@ -0,0 +1,254 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; +import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class CursorWebSocketSendLoopForegroundReconnectPolicyTest { + private static final long SEGMENT_SIZE_BYTES = 16_384L; + + @Rule + public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testPostStartAuthFailureRetriesUntilCredentialsRecover() throws Exception { + assertForegroundRecovers(false, + () -> new QwpAuthFailedException(401, "localhost", 1)); + } + + @Test + public void testPostStartDurableAckMismatchRetriesUntilCapabilityRecovers() throws Exception { + assertForegroundRecovers(true, + () -> new QwpDurableAckMismatchException("localhost", 1, "primary")); + } + + private void assertForegroundRecovers(boolean durableAck, FailureSupplier failureSupplier) throws Exception { + TestUtils.assertMemoryLeak(() -> { + DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck); + try (TestWebSocketServer server = new TestWebSocketServer(handler, true); + CursorSendEngine engine = new CursorSendEngine( + sfDir.newFolder().getAbsolutePath(), SEGMENT_SIZE_BYTES)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + WebSocketClient initialClient = connect(server.getPort(), durableAck); + ScriptedFactory factory = new ScriptedFactory( + server.getPort(), durableAck, failureSupplier); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + 100L, + 1L, + 4L, + durableAck, + durableAck ? 10L : 0L, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, + 0L, + CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND); + try { + appendFrame(engine, (byte) 1); + loop.start(); + + await(() -> loop.getTotalReconnects() >= 1L + || loop.getTerminalError() != null, + "foreground reconnect did not complete"); + Assert.assertNull("post-start endpoint-policy failures must not stop the producer", + loop.getTerminalError()); + Assert.assertTrue("the reconnect loop must retry both scripted failures", + factory.attempts() >= 3); + + long target = appendFrame(engine, (byte) 2); + await(() -> engine.ackedFsn() >= target || loop.getTerminalError() != null, + "foreground sender did not deliver after the policy failure cleared"); + Assert.assertNull(loop.getTerminalError()); + Assert.assertEquals(target, engine.ackedFsn()); + } finally { + loop.close(); + initialClient.close(); + } + } + }); + } + + private static long appendFrame(CursorSendEngine engine, byte marker) { + long buffer = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buffer + i, marker); + } + return engine.appendBlocking(buffer, 16); + } finally { + Unsafe.free(buffer, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void await(Condition condition, String failureMessage) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!condition.isTrue()) { + if (System.nanoTime() >= deadline) { + Assert.fail(failureMessage); + } + Thread.sleep(1L); + } + } + + private static WebSocketClient connect(int port, boolean durableAck) throws Exception { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + try { + client.setQwpMaxVersion(1); + client.setQwpRequestDurableAck(durableAck); + client.setConnectTimeout(5_000); + client.connect("localhost", port); + client.upgrade("/write/v4", 5_000, null); + return client; + } catch (Throwable e) { + client.close(); + throw e; + } + } + + @FunctionalInterface + private interface Condition { + boolean isTrue() throws Exception; + } + + @FunctionalInterface + private interface FailureSupplier { + Exception get(); + } + + private static final class ScriptedFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final boolean durableAck; + private final FailureSupplier failureSupplier; + private final int port; + + private ScriptedFactory(int port, boolean durableAck, FailureSupplier failureSupplier) { + this.port = port; + this.durableAck = durableAck; + this.failureSupplier = failureSupplier; + } + + int attempts() { + return attempts.get(); + } + + @Override + public WebSocketClient reconnect() throws Exception { + if (attempts.incrementAndGet() <= 2) { + throw failureSupplier.get(); + } + return connect(port, durableAck); + } + } + + private static final class DropFirstConnectionHandler + implements TestWebSocketServer.WebSocketServerHandler { + private static final String TABLE = "trades"; + private final boolean durableAck; + private final Map wireSeqByConnection = + new IdentityHashMap<>(); + private TestWebSocketServer.ClientHandler firstConnection; + + private DropFirstConnectionHandler(boolean durableAck) { + this.durableAck = durableAck; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long[] sequence = wireSeqByConnection.get(client); + if (sequence == null) { + sequence = new long[1]; + wireSeqByConnection.put(client, sequence); + if (firstConnection == null) { + firstConnection = client; + } + } + long wireSeq = sequence[0]++; + try { + client.sendBinary(okFrame(wireSeq)); + if (durableAck) { + client.sendBinary(durableAckFrame(wireSeq)); + } + if (client == firstConnection) { + client.close(); + } + } catch (IOException ignored) { + // The deliberate close may race the acknowledgement write. + } + } + + private static byte[] durableAckFrame(long seqTxn) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + ByteBuffer bb = ByteBuffer.allocate(1 + 2 + 2 + name.length + 8) + .order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x02); + bb.putShort((short) 1); + bb.putShort((short) name.length); + bb.put(name); + bb.putLong(seqTxn); + return bb.array(); + } + + private static byte[] okFrame(long wireSeq) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + ByteBuffer bb = ByteBuffer.allocate(1 + 8 + 2 + 2 + name.length + 8) + .order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(wireSeq); + bb.putShort((short) 1); + bb.putShort((short) name.length); + bb.put(name); + bb.putLong(wireSeq); + return bb.array(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java index 39755426..82f65e6b 100644 --- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java @@ -37,9 +37,15 @@ import org.junit.Assert; import org.slf4j.Logger; +import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.FileVisitResult; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; import static org.junit.Assert.assertNotNull; @@ -228,23 +234,78 @@ public static void removeTmpDirRec(String tmpDir) { if (tmpDir == null) { return; } - java.nio.file.Path root = java.nio.file.Paths.get(tmpDir); - if (!java.nio.file.Files.exists(root)) { - return; - } - // try-with-resources: the walk Stream holds an open directory handle that - // must be closed, or each call leaks a descriptor. - try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) { - walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { - try { - java.nio.file.Files.deleteIfExists(p); - } catch (java.io.IOException ignored) { + removeTmpDirRec(Paths.get(tmpDir), java.nio.file.Files::deleteIfExists); + } + + /** + * Injectable counterpart used to verify that cleanup remains best-effort but + * still fails the test when any path cannot be removed. + */ + static void removeTmpDirRec(Path root, PathDeleter deleter) { + final IOException[] firstFailure = new IOException[1]; + try { + java.nio.file.Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + remember(exc); + delete(dir); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + remember(exc); + return FileVisitResult.CONTINUE; + } + + private void delete(Path path) { + try { + deleter.delete(path); + } catch (IOException e) { + remember(e); + } + } + + private void remember(IOException failure) { + if (failure == null || failure instanceof NoSuchFileException) { + // The cleanup goal for this path is already satisfied. This + // also covers a child disappearing between directory listing + // and attribute lookup. + return; + } + if (firstFailure[0] == null) { + firstFailure[0] = failure; + } else if (firstFailure[0] != failure) { + firstFailure[0].addSuppressed(failure); + } } }); - } catch (java.io.IOException ignored) { + } catch (NoSuchFileException ignored) { + // A missing root is an already-complete cleanup. + } catch (IOException e) { + if (firstFailure[0] == null) { + firstFailure[0] = e; + } else { + firstFailure[0].addSuppressed(e); + } + } + if (firstFailure[0] != null) { + throw new AssertionError("could not recursively remove temp directory: " + root, + firstFailure[0]); } } + @FunctionalInterface + interface PathDeleter { + void delete(Path path) throws IOException; + } + /** * Java 8 stand-in for {@code String.repeat(int)} (added in Java 11). */ diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java b/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java new file mode 100644 index 00000000..61cee092 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java @@ -0,0 +1,65 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.tools; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; + +public class TestUtilsTest { + + @Test + public void testRecursiveCleanupRetainsFirstFailureAndContinuesDeleting() throws Exception { + Path root = Files.createTempDirectory("qdb-test-utils-cleanup-"); + Path child = Files.createDirectory(root.resolve("child")); + Path file = Files.createFile(child.resolve("blocked")); + IOException injected = new IOException("simulated deletion failure"); + AtomicInteger attempts = new AtomicInteger(); + try { + try { + TestUtils.removeTmpDirRec(root, path -> { + attempts.incrementAndGet(); + if (path.equals(file)) { + throw injected; + } + Files.deleteIfExists(path); + }); + Assert.fail("cleanup must report the injected deletion failure"); + } catch (AssertionError e) { + Assert.assertSame("the first deletion failure must be retained", injected, e.getCause()); + Assert.assertTrue("cleanup must continue with parent directories after a file failure", + attempts.get() >= 3); + Assert.assertTrue("later parent deletion failures must remain diagnostic", + e.getCause().getSuppressed().length >= 2); + } + } finally { + TestUtils.removeTmpDirRec(root.toString()); + } + } +}