diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4ec70dbd..be4fe409 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -38,6 +38,10 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -988,6 +992,12 @@ final class LineSenderBuilder { // build() time. 0 or negative is a documented "disable" value, so // a Long.MIN_VALUE sentinel keeps it distinguishable from "unset". private static final long DURABLE_ACK_KEEPALIVE_NOT_SET = Long.MIN_VALUE; + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(LineSenderBuilder.class); + // How many quarantined copies of one slot may pile up under sf_dir before build() + // refuses to set aside another. Each is an unreplayable slot a human still has to + // look at; accumulating them without bound would turn a disk-space problem into a + // second incident. + private static final int MAX_QUARANTINE_SLOT_ATTEMPTS = 64; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // sf-client.md section 4.4: the inbox capacity must accommodate the // distinct error categories in a bursty error stream so that drop-oldest @@ -1003,6 +1013,13 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + @TestOnly + private static volatile Runnable quarantineAfterCloseHook; + // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the + // sender's own slot name, so a restarted sender does not re-adopt it as its own; + // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and + // the bytes stay put for a human to inspect and resend. + private static final String QUARANTINE_SLOT_SUFFIX = ".unreplayable-"; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; @@ -1051,6 +1068,7 @@ final class LineSenderBuilder { private int errorInboxCapacity = PARAMETER_NOT_SET_EXPLICITLY; private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY; private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; + private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; private String httpPath; private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; @@ -1505,6 +1523,10 @@ public Sender build() { long actualPoisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY ? poisonMinEscalationWindowMillis : CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + long actualCatchUpCapGapMinEscalationWindowMillis = + catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY + ? catchUpCapGapMinEscalationWindowMillis + : CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; // sfDir is the parent (group root); the actual slot lives // under sfDir/senderId. This is what the engine sees — the @@ -1540,55 +1562,101 @@ public Sender build() { sfAppendDeadlineMillis == PARAMETER_NOT_SET_EXPLICITLY ? CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS : sfAppendDeadlineMillis * 1_000_000L; - CursorSendEngine cursorEngine = new CursorSendEngine( - slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? errorInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; - int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? connectionListenerInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; - List wsEndpoints = - new ArrayList<>(hosts.size()); - for (int i = 0, n = hosts.size(); i < n; i++) { - wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); - } - QwpWebSocketSender connected; - try { - connected = QwpWebSocketSender.connect( - wsEndpoints, - wsTlsConfig, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - wsAuthHeader, - requestDurableAck, - cursorEngine, - actualCloseFlushTimeoutMillis, - actualReconnectMaxDurationMillis, - actualReconnectInitialBackoffMillis, - actualReconnectMaxBackoffMillis, - actualInitialConnectMode, - errorHandler, - actualErrorInboxCapacity, - actualDurableAckKeepaliveIntervalMillis, - authTimeoutMillis, - connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, - connectionListener, - actualConnectionListenerInboxCapacity, - actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis - ); - } catch (Throwable t) { - // connect() failed before ownership of cursorEngine - // transferred — close it ourselves. - try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort + QwpWebSocketSender connected = null; + // The parent-anchored logical lock is stable across a slot rename. Keep it + // from before the directory-local lock is acquired until connect() has either + // adopted that engine or quarantine has closed, renamed and recreated it. + // This closes the inode-swap window in which an already-queued orphan drainer + // could otherwise acquire the renamed directory's old .lock and later operate + // on the fresh slot through the original pathname. + try (SlotLock logicalSlotLock = slotPath == null + ? null + : SlotLock.acquireLogical(slotPath)) { + CursorSendEngine cursorEngine = new CursorSendEngine( + slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? errorInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; + int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? connectionListenerInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; + List wsEndpoints = + new ArrayList<>(hosts.size()); + for (int i = 0, n = hosts.size(); i < n; i++) { + wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); + } + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + boolean quarantined = false; + while (connected == null) { + try { + connected = QwpWebSocketSender.connect( + wsEndpoints, + wsTlsConfig, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + wsAuthHeader, + requestDurableAck, + cursorEngine, + actualCloseFlushTimeoutMillis, + actualReconnectMaxDurationMillis, + actualReconnectInitialBackoffMillis, + actualReconnectMaxBackoffMillis, + actualInitialConnectMode, + errorHandler, + actualErrorInboxCapacity, + actualDurableAckKeepaliveIntervalMillis, + authTimeoutMillis, + connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, + connectionListener, + actualConnectionListenerInboxCapacity, + actualMaxFrameRejections, + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis + ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + } catch (Throwable t) { + // connect() failed before ownership of cursorEngine + // transferred — close it ourselves. + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw t; + } } - throw t; } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY @@ -1703,6 +1771,35 @@ public Sender build() { return sender; } + /** + * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must + * persist before an orphan store-and-forward drainer quarantines its slot. + *

+ * A cap gap means a symbol already accepted by one node is too large to + * re-register on the node the sender just failed over to, because that node + * advertises a smaller maximum batch size. On a homogeneous cluster this cannot + * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering + * the cap below existing data. + *

+ * A foreground sender retries such a gap indefinitely because the larger-cap node + * may simply be away. An orphan drainer may quarantine only once the gap has BOTH + * recurred many times AND persisted for this long. Raise it for a cluster whose + * rolling restarts take longer than the 5-minute default; set it to {@code 0} to + * quarantine an orphan slot as soon as the retry count is exhausted. + *

+ * WebSocket transport only. + */ + public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + if (millis < 0) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis must be >= 0: ").put(millis); + } + this.catchUpCapGapMinEscalationWindowMillis = millis; + return this; + } + /** * close() drain timeout in milliseconds. The sender's {@code close()} * method blocks up to this many millis waiting for the server to ACK @@ -2916,6 +3013,89 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na } } + /** + * Sets a recovered slot whose symbol dictionary cannot cover its surviving frames + * aside, and returns a fresh engine on an empty slot so the producer can keep + * producing. + *

+ * Such a slot is unreplayable BY THIS PRODUCER: its frames reference symbol ids the + * recovered dictionary lost (a host/power crash tore the unsynced side-file), so a + * producer seeded from the short dictionary would hand those ids to different + * symbols and silently misattribute values. Detecting that is correct and + * load-bearing -- but simply THROWING is not a safe response. {@code senderId} + * defaults to a stable name, so a restarted process re-adopts the same slot; the + * engine's close retains a slot that is not fully drained; and so every subsequent + * {@code build()} would re-recover the same slot and throw again -- forever, until + * an operator deleted the directory by hand. The application could not construct a + * Sender at all, and so could not even BUFFER new rows. That trades a bounded, + * already-lost batch for an unbounded outage of everything after it, which inverts + * the one guarantee store-and-forward exists to give. + *

+ * So: rename the slot aside instead, and mark it {@code .failed}. The verdict is + * authoritative -- the recovery seed already tried every source of truth (the + * persisted prefix AND the surviving frames' own deltas), and the orphan drainer's + * own replay guard uses that same walk, so there is nothing a drainer could rebuild + * that the seed did not. {@code markFailed} (below) therefore quarantines the copy + * for a human rather than leaving a drainer to retry an unreplayable slot forever; + * a full-dictionary-fallback slot never reaches here, because its dictionary is + * discarded at recovery and it never throws. The bytes are preserved on disk for + * forensics and a manual resend, and the new name -- NOT the sender's own slot name + * -- keeps a restarted sender from re-adopting it. The producer, meanwhile, starts + * on a clean empty slot and never notices. + *

+ * If the rename fails (a Windows share lock, a read-only mount) there is no way to + * free the slot name without destroying data, so fall back to the old behaviour and + * throw -- loudly, and never silently dropping bytes. + */ + private static CursorSendEngine quarantineTornSlot( + CursorSendEngine torn, UnreplayableSlotException cause, String sfDir, + String senderId, String slotPath, + long sfMaxBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos + ) { + // The verdict, and the reason, come from the recovery seed -- the only code that + // has tried every source of truth. Recomputing them here would mean a second, + // independently-drifting notion of "unreplayable". + final String detail = cause.getMessage(); + // Release the slot lock and the dictionary fd before renaming. connect()'s failure + // path already closed the engine; close() is idempotent, so make it explicit rather + // than depend on that. + torn.close(); + Runnable hook = quarantineAfterCloseHook; + if (hook != null) { + hook.run(); + } + + String quarantinePath = null; + for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { + String candidate = sfDir + "/" + senderId + QUARANTINE_SLOT_SUFFIX + i; + if (!Files.exists(candidate)) { + quarantinePath = candidate; + break; + } + } + if (quarantinePath == null || Files.rename(slotPath, quarantinePath) != 0) { + throw new LineSenderException( + detail + "; the affected data must be resent. The slot could not be set aside " + + "automatically (" + (quarantinePath == null + ? "too many quarantined slots already under " + sfDir + : "rename to " + quarantinePath + " failed") + + "), so this sender cannot start until " + + slotPath + " is moved or removed by hand"); + } + // Mark the quarantined copy so the orphan drainer treats it as a + // human-in-the-loop slot rather than silently retrying it forever. + OrphanScanner.markFailed(quarantinePath, detail); + LOG.error("{} -- the slot has been set aside at {} and the affected data must be resent; " + + "this sender continues on a fresh, empty slot at {}", + detail, quarantinePath, slotPath); + return new CursorSendEngine(slotPath, sfMaxBytes, sfMaxTotalBytes, sfAppendDeadlineNanos); + } + + @TestOnly + public static void setQuarantineAfterCloseHookForTest(Runnable hook) { + quarantineAfterCloseHook = hook; + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); @@ -3413,6 +3593,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "poison_min_escalation_window_millis"); poisonMinEscalationWindowMillis(parseLongValue(sink, "poison_min_escalation_window_millis")); + } else if (Chars.equals("catchup_cap_gap_min_escalation_window_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "catchup_cap_gap_min_escalation_window_millis"); + catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catchup_cap_gap_min_escalation_window_millis")); } else if (Chars.equals("initial_connect_retry", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport"); @@ -3686,6 +3872,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("poison_min_escalation_window_millis")) { poisonMinEscalationWindowMillis(wsLong(view, v, "poison_min_escalation_window_millis")); } + if (view.has("catchup_cap_gap_min_escalation_window_millis")) { + catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catchup_cap_gap_min_escalation_window_millis")); + } if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } @@ -3862,6 +4051,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("max_background_drainers", maxBackgroundDrainers); m.put("max_frame_rejections", maxFrameRejections); m.put("poison_min_escalation_window_millis", poisonMinEscalationWindowMillis); + m.put("catchup_cap_gap_min_escalation_window_millis", catchUpCapGapMinEscalationWindowMillis); m.put("error_inbox_capacity", errorInboxCapacity); m.put("connection_listener_inbox_capacity", connectionListenerInboxCapacity); m.put("token", httpToken); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 3b4c9a90..d1b0f13e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) { this.idToSymbol = new ObjList<>(initialCapacity); } + /** + * Appends {@code symbol} at the next sequential id, matching a recovered / + * persisted dictionary's dense id order, WITHOUT de-duplicating. + *

+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted}) + * replays the persisted entries in id order to rebuild this dictionary. It must + * NOT collapse two source strings that decode to the same characters, because + * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread + * catch-up mirror all key on the entry POSITION (id), not on the string. The + * only strings that collide this way are malformed lone UTF-16 surrogates, + * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would + * de-dup them and leave this dictionary SHORTER than the persisted entry count, + * desyncing the producer's delta baseline from the catch-up mirror (which uses + * {@code pd.size()}) and silently misattributing later symbols. Appending + * unconditionally keeps {@link #size()} equal to that count. The reverse lookup + * keeps the highest id for a colliding string, which is harmless: both ids + * encode to the same bytes, so resolving either is equivalent. + * + * @param symbol the recovered symbol string (must not be null) + * @return the id assigned (the previous {@link #size()}) + */ + public int addRecoveredSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + /** * Clears all symbols from the dictionary. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index aad95f3c..86343772 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -76,6 +76,27 @@ public static int varintSize(long value) { return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; } + /** + * Writes {@code value} as an unsigned LEB128 varint directly at native address + * {@code addr} and returns the address just past the last byte. The canonical + * raw-address varint writer shared by the SF cursor's persisted dictionary and + * catch-up frame builder. + *

+ * {@code value} must be non-negative: the signed {@code value > 0x7F} loop emits + * a SINGLE truncated byte for a negative long, whereas {@link #varintSize} + * returns 10 for it -- a size/write mismatch that would corrupt the stream. All + * callers pass ids/lengths/counts (non-negative); the assert pins that contract. + */ + public static long writeVarint(long addr, long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + @Override public void close() { if (bufferPtr != 0) { @@ -305,6 +326,7 @@ public void putUtf8(CharSequence value) { */ @Override public void putVarint(long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; ensureCapacity(10); // max varint bytes long addr = bufferPtr + position; while (value > 0x7F) { @@ -336,11 +358,7 @@ public void skip(int bytes) { } private static void writeVarintDirect(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr, (byte) value); + writeVarint(addr, value); } private void encodeUtf8(CharSequence value, int utf8Len) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ced1a1b5..f38d9e93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; @@ -39,6 +41,16 @@ public class QwpWebSocketEncoder implements QuietCloseable { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; + // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region + // ([len][utf8]... only, without the two section varints) that beginMessage + // last wrote. Let the producer persist those bytes straight to the slot's + // .symbol-dict instead of re-encoding the same symbols (see + // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next + // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaCount; + private int deltaEntriesEnd; + private int deltaEntriesStart; + private int deltaStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -65,8 +77,8 @@ public void beginMessage( int batchMaxId ) { buffer.reset(); - int deltaStart = confirmedMaxId + 1; - int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + deltaStart = confirmedMaxId + 1; + deltaCount = Math.max(0, batchMaxId - confirmedMaxId); byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); byte origFlags = flags; flags = headerFlags; @@ -75,10 +87,12 @@ public void beginMessage( payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); + deltaEntriesStart = buffer.getPosition(); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } + deltaEntriesEnd = buffer.getPosition(); columnWriter.setBuffer(buffer); } @@ -90,6 +104,64 @@ public void close() { } } + /** + * Copies one single-table split message from the combined message currently + * staged in this encoder. The table body is copied byte-for-byte from its + * recorded offset; columns and rows are not encoded again. + */ + public int copySplitMessage( + MicrobatchBuffer target, + int tableBodyOffset, + int tableBodyLength, + boolean deferCommit, + int confirmedMaxId, + int batchMaxId + ) { + if (target.getBufferPos() != 0) { + throw new IllegalStateException("split message target is not empty"); + } + if (tableBodyOffset < deltaEntriesEnd + || tableBodyLength < 0 + || (long) tableBodyOffset + tableBodyLength > buffer.getPosition()) { + throw new IllegalArgumentException("table body slice is outside the staged message"); + } + + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + int messageSize = splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + target.ensureCapacity(messageSize); + + long source = buffer.getBufferPtr(); + long destination = target.getBufferPtr(); + Vect.memcpy(destination, source, HEADER_SIZE); + + byte splitFlags = Unsafe.getUnsafe().getByte(source + HEADER_OFFSET_FLAGS); + if (deferCommit) { + splitFlags |= FLAG_DEFER_COMMIT; + } else { + splitFlags &= ~FLAG_DEFER_COMMIT; + } + Unsafe.getUnsafe().putByte(destination + HEADER_OFFSET_FLAGS, splitFlags); + Unsafe.getUnsafe().putShort(destination + 6, (short) 1); + Unsafe.getUnsafe().putInt(destination + 8, messageSize - HEADER_SIZE); + + long writeAddress = destination + HEADER_SIZE; + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaStart); + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaCount); + if (deltaEntriesLength > 0) { + Vect.memcpy(writeAddress, source + deltaEntriesStart, deltaEntriesLength); + writeAddress += deltaEntriesLength; + } + Vect.memcpy(writeAddress, source + tableBodyOffset, tableBodyLength); + writeAddress += tableBodyLength; + assert writeAddress == destination + messageSize; + + target.setBufferPos(messageSize); + return messageSize; + } + public int encode(QwpTableBuffer tableBuffer) { buffer.reset(); writeHeader(1, 0); @@ -122,6 +194,33 @@ public QwpBufferWriter getBuffer() { return buffer; } + /** + * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...}, + * excluding the two section varints) that {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesLen() { + return deltaEntriesEnd - deltaEntriesStart; + } + + /** + * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY + * region {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesStart() { + return deltaEntriesStart; + } + + public int getSplitMessageSize(int tableBodyLength, int confirmedMaxId, int batchMaxId) { + if (tableBodyLength < 0) { + throw new IllegalArgumentException("tableBodyLength must be non-negative"); + } + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + return splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; @@ -144,4 +243,35 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { + if (splitDeltaCount == 0) { + return 0; + } + if (splitDeltaStart != deltaStart || splitDeltaCount != deltaCount) { + throw new IllegalStateException("split delta does not match the staged message" + + " [stagedStart=" + deltaStart + + ", stagedCount=" + deltaCount + + ", splitStart=" + splitDeltaStart + + ", splitCount=" + splitDeltaCount + ']'); + } + return deltaEntriesEnd - deltaEntriesStart; + } + + private int splitMessageSize( + int tableBodyLength, + int splitDeltaStart, + int splitDeltaCount, + int deltaEntriesLength + ) { + long messageSize = (long) HEADER_SIZE + + NativeBufferWriter.varintSize(splitDeltaStart) + + NativeBufferWriter.varintSize(splitDeltaCount) + + deltaEntriesLength + + tableBodyLength; + if (messageSize > Integer.MAX_VALUE) { + throw new OutOfMemoryError("split QWP message size overflow: " + messageSize); + } + return (int) messageSize; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..557af871 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -46,6 +46,8 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; @@ -56,6 +58,7 @@ import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.IntList; import io.questdb.client.std.Misc; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -168,6 +171,11 @@ public class QwpWebSocketSender implements Sender { // behind a drainer's endpoint walk. private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; + // Per-table encoded body byte counts captured during flushPendingRows' combined + // encode. flushPendingRowsSplit uses them both for preflight sizing and to walk + // the staged body slices without encoding the batch a second time. Cleared and + // repopulated on every flush; reused to stay zero-GC. + private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -221,6 +229,17 @@ public class QwpWebSocketSender implements Sender { private CursorSendEngine cursorEngine; private CursorWebSocketSendLoop cursorSendLoop; private boolean deferCommit; + // True when the sender emits incremental (delta) symbol dictionaries: each + // message carries only symbol ids not yet sent on the wire, rather than the + // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from + // the in-process ring) and in file-mode store-and-forward when the per-slot + // persisted dictionary opened. In both, the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying, so a non-self-sufficient + // delta frame never dangles an id on a fresh server. Falls back to full + // self-sufficient frames only when the persisted dictionary is unavailable in + // file-mode (recovery/orphan-drain would then have nothing to rebuild the + // deltas from). Set in setCursorEngine. + private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after // startOrphanDrainers) and read at pool-creation time. Null -> drainers @@ -295,6 +314,12 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before an + // orphan drainer may quarantine its slot (connect-string key + // catchup_cap_gap_min_escalation_window_millis). Foreground senders retry forever. See + // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. + private long catchUpCapGapMinEscalationWindowMillis = + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; private long reconnectInitialBackoffMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS; private long reconnectMaxBackoffMillis = @@ -313,6 +338,12 @@ public class QwpWebSocketSender implements Sender { // beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the // first round indicates the initial connect. private long roundSeq; + // Highest global symbol id the producer has baked into a frame so far, or -1. + // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because + // the I/O thread re-registers the full dictionary via a catch-up frame before + // replaying, so the producer's delta baseline stays valid across the wire + // boundary. Used only when deltaDictEnabled; ignored in full-dict mode. + private int sentMaxSymbolId = -1; // When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only // explicit flush() triggers the server-side commit. Enables accumulating // arbitrarily large datasets that exceed the server's recv buffer. @@ -667,7 +698,47 @@ public static QwpWebSocketSender connect( durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, connectionListenerInboxCapacity, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); + } + + /** + * Compatibility overload retained for callers compiled before the + * catch-up cap-gap dwell was added to the master signature. + */ + public static QwpWebSocketSender connect( + List endpoints, + ClientTlsConfiguration tlsConfig, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + String authorizationHeader, + boolean requestDurableAck, + CursorSendEngine cursorEngine, + long closeFlushTimeoutMillis, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + Sender.InitialConnectMode initialConnectMode, + SenderErrorHandler errorHandler, + int errorInboxCapacity, + long durableAckKeepaliveIntervalMillis, + long authTimeoutMs, + int connectTimeoutMs, + SenderConnectionListener connectionListener, + int connectionListenerInboxCapacity, + int maxFrameRejections, + long poisonMinEscalationWindowMillis + ) { + return connect(endpoints, tlsConfig, autoFlushRows, autoFlushBytes, + autoFlushIntervalNanos, authorizationHeader, requestDurableAck, + cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, + initialConnectMode, errorHandler, errorInboxCapacity, + durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, + connectionListener, connectionListenerInboxCapacity, + maxFrameRejections, poisonMinEscalationWindowMillis, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -698,7 +769,8 @@ public static QwpWebSocketSender connect( SenderConnectionListener connectionListener, int connectionListenerInboxCapacity, int maxFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -716,6 +788,7 @@ public static QwpWebSocketSender connect( sender.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; sender.maxFrameRejections = maxFrameRejections; sender.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + sender.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; @@ -1762,6 +1835,38 @@ public long getPendingBytes() { return pendingBytes; } + /** + * Snapshot of the producer's symbol prefix whose persisted-dictionary chunks + * have committed. The persisted size advances only after the chunk CRC and + * payload have been written, so this observes the write-ahead boundary without + * reopening the live mmap-backed dictionary (which would attempt recovery-tail + * truncation and is not supported while the producer owns the file on Windows). + * Returns {@code null} in memory mode or when the persisted dictionary is + * unavailable. + */ + @TestOnly + public ObjList getPersistedSymbolsForTest() { + CursorSendEngine engine = cursorEngine; + if (engine == null) { + return null; + } + PersistedSymbolDict persisted = engine.getPersistedSymbolDict(); + if (persisted == null) { + return null; + } + int persistedSize = persisted.size(); + int globalSize = globalSymbolDictionary.size(); + if (persistedSize > globalSize) { + throw new IllegalStateException("persisted symbol dictionary exceeds producer dictionary" + + " [persisted=" + persistedSize + ", producer=" + globalSize + ']'); + } + ObjList snapshot = new ObjList<>(persistedSize); + for (int i = 0; i < persistedSize; i++) { + snapshot.add(globalSymbolDictionary.getSymbol(i)); + } + return snapshot; + } + /** * Server-advertised cap on the per-batch raw byte size. Zero before the * first connect; updated by every successful reconnect via @@ -2215,6 +2320,22 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } this.cursorEngine = engine; this.ownsCursorEngine = takeOwnership && engine != null; + // Delta encoding is available in memory-mode (in-process catch-up) and in + // file-mode when the persisted dictionary opened (recovery / orphan-drain + // rebuild the dictionary from it). Otherwise fall back to full self- + // sufficient frames. See CursorSendEngine.isDeltaDictEnabled. + this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled(); + // Recovery: repopulate the producer's global dictionary from the slot's + // persisted dictionary so newly ingested symbols continue from the + // recovered ids (rather than colliding with them at 0), and the delta + // baseline resumes where the crashed session left off. + // NOT gated on deltaDictEnabled. That flag is false exactly when the slot's dictionary + // failed to open -- which is precisely when the frames on disk are the only surviving + // copy of the symbols and the rebuild matters most. Gating the seed on it made the + // rebuild dead code for the very case it was written for. + if (engine != null && engine.wasRecoveredFromDisk()) { + seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); + } } /** @@ -2378,7 +2499,8 @@ public synchronized void startOrphanDrainers( requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); ref[0] = drainer; drainerPool.submit(drainer); } @@ -3306,7 +3428,9 @@ private void ensureConnected() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles @@ -3332,6 +3456,17 @@ private void ensureConnected() { cursorSendLoop.setConnectionDispatcher(connectionDispatcher); cursorSendLoop.start(); } catch (Throwable t) { + // start() (or dispatcher construction) failed after cursorSendLoop was + // assigned. Close it so a caller that retries -- re-entering + // ensureConnected and reassigning cursorSendLoop above -- cannot orphan + // a recovered slot's ctor-seeded native mirror (freed only by close() + // or the I/O loop, neither of which has run). close() is idempotent and + // frees the mirror via its loopNeverRan path; it also closes the shared + // client, so the client.close() below is a safe idempotent no-op. + if (cursorSendLoop != null) { + cursorSendLoop.close(); + cursorSendLoop = null; + } if (client != null) { client.close(); client = null; @@ -3359,18 +3494,18 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). The symbol-dict watermark still gets - // reset for consistency with the sync path; the post-connect replay - // path does not need a producer-side reset signal because every - // cursor frame is self-sufficient. + // its default version (V1). The per-batch symbol-dict watermark still + // gets reset for consistency with the sync path; the post-connect + // replay path needs no producer-side reset signal (see below). Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection, so reset the symbol-dict - // watermark. Cursor frames are self-sufficient (every frame carries its - // full inline schema + a symbol-dict delta from id 0), so post-reconnect - // replay needs no producer-side reset signal. + // Server starts fresh on each connection, so reset the per-batch + // symbol-dict watermark. Every frame still carries its full inline schema, + // and the fresh server's dictionary is re-established either by a full-dict + // frame (full-dict mode) or by an I/O-thread catch-up frame before replay + // (delta mode), so post-reconnect replay needs no producer-side reset signal. resetSymbolDictStateForNewConnection(); connectionError.set(null); @@ -3423,14 +3558,23 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient: - // recorded frames replay to fresh server connections (orphan-slot - // drainers and post-reconnect replay), so always emit the full - // symbol-dict delta from id=0 and the full column schema inline, - // never a back-reference the target server may not have seen. + // In full-dict mode every frame is self-sufficient: it carries the whole + // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh + // server never dangles a symbol id. In delta mode (memory-mode, and + // file-mode store-and-forward once the persisted dictionary opened) each + // frame carries only ids above sentMaxSymbolId; a reconnect re-registers + // the dictionary via an I/O-thread catch-up frame before replay, so the + // producer's monotonic baseline stays valid across the wire boundary. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); + // Record each table's encoded body size (position delta across addTable). + // When the batch needs splitting, these lengths delimit immutable body + // slices in the combined encoder buffer for direct frame assembly. The + // capture is a couple of int ops per table on the common path. + splitFrameBodyBytes.clear(); + int combinedBodyStart = encoder.getBuffer().getPosition(); + int bodyStart = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3447,19 +3591,40 @@ private void flushPendingRows(boolean deferCommit) { } encoder.addTable(tableBuffer); + int bodyEnd = encoder.getBuffer().getPosition(); + splitFrameBodyBytes.add(bodyEnd - bodyStart); + bodyStart = bodyEnd; } int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { - flushPendingRowsSplit(keys, deferCommit); + // Snapshot the volatile cap ONCE for this whole flush. The I/O thread lowers + // serverMaxBatchSize on a mid-stream failover to a smaller-cap node + // (applyServerBatchSizeLimit); if the split pre-flight and the publish loop + // re-read the field independently, a failover between them would size frames + // against two different caps -- breaking the all-or-nothing guarantee and + // firing the publish-loop assert on a legitimate race. Both use this snapshot; + // the next flush picks up the new cap. + int cap = serverMaxBatchSize; + if (cap > 0 && messageSize > cap) { + // Keep the completed combined frame staged in the encoder while the + // split path copies its delta entries and table-body slices. + flushPendingRowsSplit(keys, deferCommit, combinedBodyStart, cap); return; } + // Write-ahead: durably persist this frame's new symbols BEFORE it is + // published, so a recovered/orphan-drained slot can always rebuild the + // dictionary the (non-self-sufficient) delta frame references. No-op in + // memory mode and when the frame introduces no new symbols. + persistNewSymbolsBeforePublish(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring; + // advance the delta baseline so the next frame ships only newer ids. + advanceSentMaxSymbolId(); hasDeferredMessages = deferCommit; if (!deferCommit) { @@ -3475,30 +3640,84 @@ private void flushPendingRows(boolean deferCommit) { * own message. All messages except the last carry FLAG_DEFER_COMMIT * so the server appends rows without committing until the final * message arrives. + *

+ * Not atomic across frames. The frames publish one at a time, so a + * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on + * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout -- + * leaves frames 1..k-1 already on the ring as deferred (appended, not yet + * committed). The throw propagates past the {@code resetTableBuffersAfterFlush} + * at the end of the loop, so the source rows survive in their table buffers + * and the NEXT flush re-emits the whole batch; the eventual commit then + * commits the already-published prefix alongside the re-sent copies, + * delivering those rows at-least-once (duplicated), not exactly-once. This is + * within store-and-forward's at-least-once contract -- a DEDUP table or a + * durable-ack await absorbs the duplicate, and the symbol-dict state stays + * consistent on the retry (the re-sent frames carry empty deltas and the + * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic + * (rolling back the published prefix, or skipping it on retry) would be a + * larger change. * * @param deferCommit when true, ALL messages (including the last) * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) { + private void flushPendingRowsSplit( + ObjList keys, + boolean deferCommit, + int combinedBodyStart, + int cap + ) { if (LOG.isDebugEnabled()) { - LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); - } - - // Collect non-empty table indices so we know which is last. + LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); + } + + // Collect non-empty table indices so we know which is last, AND pre-flight + // every split frame's size BEFORE publishing any of them. The split hands + // frames to the ring one at a time (all but the last deferred -- appended but + // uncommitted); if a later table's frame were only found oversized + // mid-publish, the already-published prefix would strand on the ring, a + // subsequent commit would deliver it as a partial batch, and + // resetTableBuffersAfterFlush would discard every source row -- a partial + // commit the caller was told (by the throw) had failed. Checking all sizes up + // front makes the split all-or-nothing: either every frame fits and all + // publish, or none publish and we throw with nothing stranded. + // + // Each split frame's size is derived from the combined encode flushPendingRows + // already performed. simBaseline mirrors the publish loop's baseline advance + // (advanceSentMaxSymbolId), so each size equals the frame the staged-slice + // assembler will build; this pass mutates no delta/persist state. int nonEmptyCount = 0; + int simBaseline = symbolDeltaBaseline(); + int bodyIdx = 0; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { continue; } QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - nonEmptyCount++; + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + nonEmptyCount++; + int messageSize = encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(bodyIdx), simBaseline, currentBatchMaxSymbolId); + bodyIdx++; + if (messageSize > cap) { + throw new LineSenderException("single table batch too large for server batch cap") + .put(" [table=").put(tableName) + .put(", messageSize=").put(messageSize) + .put(", serverMaxBatchSize=").put(cap).put(']'); + } + // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's + // new ids, the remaining frames carry an empty delta above the baseline. + if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { + simBaseline = currentBatchMaxSymbolId; } } int sent = 0; + bodyIdx = 0; + int tableBodyOffset = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3513,26 +3732,43 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm boolean isLast = (sent == nonEmptyCount); boolean deferThis = deferCommit || !isLast; - encoder.setDeferCommit(deferThis); - encoder.beginMessage(1, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); - QwpBufferWriter buffer = encoder.getBuffer(); - - if (messageSize > serverMaxBatchSize) { - resetTableBuffersAfterFlush(keys); - throw new LineSenderException("single table batch too large for server batch cap") - .put(" [table=").put(tableName) - .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); - } - + int tableBodyLength = splitFrameBodyBytes.getQuick(bodyIdx++); + // Persist before touching activeBuffer. If the write-ahead fails, the + // caller can retry with both the source rows and active microbatch + // unchanged. The first frame carries the batch's new symbols; later + // frames are no-ops once the baseline has advanced. + persistNewSymbolsBeforePublish(); ensureActiveBufferReady(); - activeBuffer.ensureCapacity(messageSize); - activeBuffer.write(buffer.getBufferPtr(), messageSize); + // The combined encoder buffer remains immutable for the whole split. + // Assemble this frame directly into the active microbatch: patched + // header + staged delta bytes + the staged table-body slice. No row or + // column is encoded a second time. + int messageSize = encoder.copySplitMessage( + activeBuffer, + tableBodyOffset, + tableBodyLength, + deferThis, + symbolDeltaBaseline(), + currentBatchMaxSymbolId + ); + tableBodyOffset += tableBodyLength; + // The pre-flight pass above already verified every split frame fits the + // cap, so none can be found oversized here -- which is what keeps this + // loop from publishing (and stranding) a deferred prefix before an + // oversized table. Both passes size against the SAME snapshot cap, so a + // mid-flush failover cannot make them disagree; the assert therefore only + // catches a genuine divergence between the pre-flight arithmetic and the + // staged assembler (a future bug), not a cap race. It deliberately does NOT + // reset+throw here, because by this point a prefix may already be on the ring. + assert messageSize <= cap + : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; + activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // Frame queued: advance so the next split frame's delta starts above + // the ids this one just registered. + advanceSentMaxSymbolId(); } encoder.setDeferCommit(false); @@ -3573,8 +3809,25 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); + // A commit carries no rows, and it must also carry NO new symbols. Unlike + // the flush paths, sendCommitMessage does NOT write-ahead-persist the + // dictionary, so shipping a symbol here would put an id on the wire that a + // recovered slot cannot rebuild from the persisted .symbol-dict, diverging + // the producer dictionary from the surviving frames and silently + // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit + // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back + // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at + // what has already been sent -- and therefore already persisted. In delta + // mode pass sentMaxSymbolId, yielding an empty delta + // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep + // currentBatchMaxSymbolId (reset to -1 by the prior flush, so the commit frame + // carries an empty delta too -- harmless, since the preceding full-dict data + // frames already registered the dictionary on this connection). Any symbol a + // cancelled row leaked is picked up (and persisted) by the next real flush, + // whose persistNewSymbolsBeforePublish resumes from pd.size(). + int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; encoder.beginMessage(0, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), commitBatchMaxId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); @@ -3586,15 +3839,219 @@ private void sendCommitMessage() { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + /** + * Advances the delta baseline once a frame carrying the current batch's + * symbols has been queued onto the ring. No-op in full-dict mode. Only ever + * moves the baseline forward, so a batch that used no new symbols leaves it + * unchanged. + */ + private void advanceSentMaxSymbolId() { + if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) { + sentMaxSymbolId = currentBatchMaxSymbolId; + } + } + + /** + * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 .. + * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the + * frame is published to the ring. This write-ahead ordering keeps the + * persisted dictionary a superset of every process-crash-recoverable frame's + * references, so recovery and orphan-drain can re-register it on a fresh + * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it + * is caught by the send loop's replay guard. No-op in memory mode (no + * persisted dictionary) and when the frame introduces no new symbols. + */ + private void persistNewSymbolsBeforePublish() { + if (!deltaDictEnabled || cursorEngine == null) { + return; + } + PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict(); + if (pd == null) { + return; + } + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the + // frame is published. + // + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: + // the persist advances pd.size() only after a full write, whereas + // sentMaxSymbolId only advances after the WHOLE frame is published (via + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist + // threw (short write -- disk full/quota) or the publish threw, the frame + // was not published and sentMaxSymbolId stayed put, while the symbols + // before the failure are already on disk. Keying the resume point off + // sentMaxSymbolId+1 would re-append that persisted prefix on the retry, + // duplicating entries and corrupting the dense id->symbol mapping recovery + // relies on (position i must be symbol id i). pd.size() resumes exactly + // past what is already durable, so the write-ahead is idempotent. + int from = pd.size(); + if (currentBatchMaxSymbolId < from) { + return; // nothing new to persist (warm batch, or an idempotent retry) + } + // Fast path: the frame the encoder just built already holds these symbols + // in its delta section as [len][utf8]... -- byte-identical to what + // PersistedSymbolDict stores. In the common case pd.size() equals the + // frame's delta start id (sentMaxSymbolId+1), so persist those bytes + // straight from the frame instead of re-encoding the symbols. After a + // failed publish the durable size has run ahead of the wire baseline, so + // the frame's delta covers MORE than remains to persist; then re-encode + // just the [from .. currentBatchMaxSymbolId] suffix. + try { + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } + } catch (Throwable t) { + // A short write (disk full / quota) to the persisted dictionary throws + // a low-level IllegalStateException. Surface it as a LineSenderException + // -- like every other flush-path failure, e.g. the cursor append in + // sealAndSwapBuffer -- so a caller catching LineSenderException around + // flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the short + // write, so the still-buffered rows re-persist the same range + // idempotently on retry. A JVM Error is never a persist failure; let it + // propagate. + if (t instanceof Error) { + throw (Error) t; + } + throw new LineSenderException("failed to persist symbol dictionary before publish", t); + } + } + private void resetSymbolDictStateForNewConnection() { - // The new server has an empty symbol dictionary, so the next batch - // must ship a delta starting at id 0. beginMessage() always passes - // confirmedMaxId = -1; resetting the batch watermark here keeps a - // stale value from suppressing re-emission of symbol ids the new - // server has never seen. + // Runs on the foreground (initial) connect only -- NOT on the I/O thread's + // reconnect/failover path. The per-batch watermark is drained state, so + // clearing it here is harmless. sentMaxSymbolId is deliberately left + // untouched: in delta mode the I/O thread re-registers the whole + // dictionary with a catch-up frame on reconnect, so the producer's + // monotonic baseline must survive the wire boundary; resetting it would + // desync the producer from the I/O thread's sent-dictionary count. currentBatchMaxSymbolId = -1; } + /** + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} so that newly + * ingested symbols continue ABOVE every id the surviving frames already define, and + * resumes the delta baseline at that tip. + *

+ * Seeds from TWO sources, in this order: + *

    + *
  1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
    2. + *
  2. the surviving frames' OWN delta sections, for every id above that prefix + * ({@link CursorSendEngine#addRecoveredSymbolsTo}).
    3. + *
+ * Those are exactly the two sources, in exactly the order, that the send loop's mirror + * is built from: its constructor seeds {@code sentDictCount} from the same dictionary, + * and {@code accumulateSentDict} then extends it from the same frames as they replay. So + * the producer's {@code sentMaxSymbolId + 1} and the loop's {@code sentDictCount} land on + * the same number BY CONSTRUCTION -- the invariant the torn-dictionary guard rests on -- + * rather than by the two happening to agree. + *

+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): the + * persisted dictionary, the on-wire delta and the mirror all key on the entry POSITION + * (id), so the producer's id space must match the recovered entry count exactly. + * {@code getOrAddSymbol} would collapse two source strings that decode to the same + * characters -- only malformed lone UTF-16 surrogates, which UTF-8-encode to {@code '?'} + * -- leaving this dictionary SHORTER than the count and silently misattributing later + * symbols. + *

+ * Why seeding from the frames matters. The dictionary is not fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can tear off its newest entries + * while the segment frames that introduced those ids survive -- and those newest frames, + * being the least likely to be acked, are exactly the ones that replay. Seeded from the + * short dictionary alone, this producer would hand its next new symbol an id those frames + * already define, putting two symbols on one id and silently misattributing values. The + * old code detected that and threw, which was safe but far too blunt: it bricked + * {@code build()} for slots the background drainer replays PERFECTLY, because the frames + * carry the torn-off symbols in their own deltas and {@code accumulateSentDict} rebuilds + * the dictionary from them. This method now rebuilds the producer from the same bytes, + * so a torn -- or entirely lost -- dictionary is recoverable whenever the surviving + * frames define the ids themselves. The next flush's write-ahead persist then re-writes + * those ids (it resumes from {@code pd.size()}), healing the side-file on disk. + *

+ * What still fails clean. A genuine GAP: the ids below a surviving frame's delta + * start were introduced by frames that were acked and TRIMMED away, so they lived only in + * the lost dictionary and nothing can rebuild them. + * {@code addRecoveredSymbolsTo} returns -1 for that and we throw. It is the same + * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so + * producer and drainer now agree on exactly which slots are recoverable, instead of the + * producer rejecting slots the drainer drains. + */ + private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + if (cursorEngine == null) { + return; + } + // 1. The dictionary's intact prefix. addRecoveredSymbol appends without de-dup, so + // the producer's size tracks pd.size() exactly -- which is what the send loop's + // mirror also seeds sentDictCount from. + int baseline = 0; + if (pd != null && pd.size() > 0) { + pd.addLoadedSymbolsTo(globalSymbolDictionary); + baseline = globalSymbolDictionary.size(); + } + // 2. Everything the surviving frames define above that prefix, straight out of their + // own delta sections -- the same bytes, in the same order, accumulateSentDict will + // feed the mirror as those frames go back on the wire. + long coverage = cursorEngine.addRecoveredSymbolsTo(baseline, globalSymbolDictionary); + if (coverage < 0) { + // A gap: the surviving frames reference ids below their own delta start, + // introduced by frames since acked and trimmed away, and the persisted + // dictionary no longer holds them (a host crash tore its unsynced tail, or it + // could not be opened). That gap only matters for frames that will REPLAY. + // When every recovered committed frame is already acked + // (ackedFsn >= recoveredCommitBoundaryFsn), NOTHING replays: the gap is in + // data the server already has, and the retired orphan-deferred tail above the + // commit boundary is never transmitted. Throwing here would raise a false + // "resend required" for delivered data AND -- because such a slot is fully + // drained -- let build()'s connect-path close unlink the (already-delivered) + // bytes the quarantine claims to preserve. So DON'T throw: seed the intact + // prefix only; addRecoveredSymbolsTo adds nothing on a -1 exactly as the + // send loop's mirror does, so the producer baseline and the mirror's + // sentDictCount still agree by construction. The producer resumes above the + // prefix and the fully-drained slot is cleaned up on close. + long ackedFsn = cursorEngine.ackedFsn(); + long commitBoundaryFsn = cursorEngine.recoveredCommitBoundaryFsn(); + if (ackedFsn >= commitBoundaryFsn) { + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + LOG.info("recovered store-and-forward slot has a torn/incomplete symbol dictionary, " + + "but every committed frame was already acked so nothing needs replaying; " + + "resuming on the intact prefix without quarantine and without data loss " + + "[recoveredPrefixSize={}, ackedFsn={}, commitBoundaryFsn={}]", + baseline, ackedFsn, commitBoundaryFsn); + return; + } + // Genuine loss: unacked committed frames reference ids nothing still holds. + // Typed, because Sender.build() sets such a slot aside instead of failing: this + // is the point at which every source of truth has been tried and none of them + // holds the missing ids. See UnreplayableSlotException. + throw new UnreplayableSlotException( + "recovered store-and-forward symbol dictionary is incomplete and cannot be " + + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + + "tail): the frames reference symbol ids below their own delta start, which " + + "were introduced by frames since acked and trimmed away, so nothing still " + + "holds them; the recovered dictionary holds only " + + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data"); + } + // Producer baseline == the coverage the replay will establish == the mirror's + // sentDictCount once those frames have gone out. The first new frame therefore + // starts its delta exactly at the tip, and the replay guard passes. + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + } + + /** + * The symbol id below which the server already holds every dictionary entry, + * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is + * the producer's monotonic sent watermark; in full-dict mode it is -1 so every + * frame re-ships the dictionary from id 0. + */ + private int symbolDeltaBaseline() { + return deltaDictEnabled ? sentMaxSymbolId : -1; + } + private void rollbackRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); @@ -3656,7 +4113,7 @@ private void sealAndSwapBuffer() { // back to it; flushPendingRows aborts its post-enqueue state // updates after this throw, so the source rows stay intact and the // next batch re-emits the same rows along with the full inline - // schema and symbol-dict delta from id 0. + // schema and the symbol-dict delta the batch requires. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index d79080d4..03ebee3e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -44,7 +44,9 @@ *

* Lifecycle: *

    - *
  1. Acquire the slot's {@code .lock}; skip silently on contention.
    2. + *
  2. Acquire the parent-anchored logical-slot lock and revalidate the + * scanner snapshot; skip silently on contention or a stale snapshot.
    3. + *
  3. Acquire the slot's {@code .lock}, then release the logical lock.
    4. *
  4. Open a {@link CursorSendEngine} on the slot — recovery picks up * every {@code .sfa} file already on disk.
    5. *
  5. Open a fresh {@link WebSocketClient} via the supplied factory @@ -132,6 +134,11 @@ public final class BackgroundDrainer implements Runnable { // Minimum wall-clock dwell before poison escalation, forwarded to every // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this + // orphan drainer's send loop latches a terminal. Foreground senders retry forever; + // this bounded policy is permitted only so a persistent orphan slot can be + // quarantined for operator intervention. + private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( String slotPath, @@ -149,7 +156,33 @@ public BackgroundDrainer( reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); + } + + /** + * Compatibility constructor retained for callers compiled before the + * catch-up cap-gap dwell was added to the master signature. + */ + public BackgroundDrainer( + String slotPath, + long segmentSizeBytes, + long sfMaxTotalBytes, + CursorWebSocketSendLoop.ReconnectFactory clientFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean requestDurableAck, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis + ) { + this(slotPath, segmentSizeBytes, sfMaxTotalBytes, clientFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, requestDurableAck, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -169,7 +202,8 @@ public BackgroundDrainer( boolean requestDurableAck, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; @@ -182,6 +216,7 @@ public BackgroundDrainer( this.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + this.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; } /** @@ -194,7 +229,7 @@ public BackgroundDrainer( @TestOnly public BackgroundDrainer() { this(null, 0L, 0L, null, 0L, 0L, 0L, false, 0L, - CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L); + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L, 0L); } /** @@ -208,8 +243,10 @@ public BackgroundDrainer() { * durable ack -- i.e. the symptom of a misconfigured cluster or a * rolling-upgrade transient. *

    - * For the foreground sender that condition is loud-fail: the producer - * is actively pushing data. The drainer is asymmetric: source data is + * For a foreground sender's initial connection that condition is loud-fail; + * after the sender has been live, its reconnect loop keeps buffering and + * retrying through a rolling capability change. The drainer is asymmetric: + * source data is * pinned (durable-ack-mode trims only on STATUS_DURABLE_ACK frames, * which the offending endpoints by definition do not send), so we * give the cluster a budget to settle before quarantining the slot. @@ -283,8 +320,7 @@ public WebSocketClient connectWithDurableAckRetry() { } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // Genuinely non-retriable across the cluster (auth 401/403, or a // non-421 upgrade reject): waiting will not fix it, so quarantine - // immediately -- exactly as the live sender's background loop - // (CursorWebSocketSendLoop.connectLoop) halts on these errors. + // immediately under the orphan reconnect policy. String msg = e.getMessage(); LOG.error("drainer terminal upgrade/auth error for slot {}: {}", slotPath, msg); lastErrorMessage = msg; @@ -521,27 +557,54 @@ private boolean stopRequestedOrInterrupted() { @Override public void run() { runnerThread = Thread.currentThread(); + SlotLock logicalSlotLock = null; CursorSendEngine engine = null; WebSocketClient client = null; CursorWebSocketSendLoop loop = null; try { - // The engine acquires the slot's .lock itself — we don't need - // (and must not) double-lock it. If another sender or drainer - // holds it, the engine constructor throws and we exit silently - // (no .failed sentinel — contention is expected, not an error). + // Scanner results are only snapshots. Serialize adoption against + // a producer's close -> quarantine rename -> fresh-slot recreate + // transition, then revalidate while that stable parent-anchored + // lock is held. The slot's own .lock inode moves with a rename and + // cannot provide this guarantee by itself. + if (slotPath != null) { + try { + logicalSlotLock = SlotLock.acquireLogical(slotPath); + } catch (IllegalStateException t) { + if (isLockContention(t)) { + LOG.info("orphan logical slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } + throw t; + } + if (!OrphanScanner.isCandidateOrphan(slotPath)) { + LOG.info("orphan candidate changed before adoption, skipping: {}", slotPath); + outcome = DrainOutcome.SUCCESS; + return; + } + } + + // The engine acquires the directory-local .lock itself. Keep the + // lock order logical -> local, and release the short-lived logical + // lock only after the engine has secured stable ownership. try { engine = new CursorSendEngine(slotPath, segmentSizeBytes, sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); } catch (IllegalStateException t) { - String msg = t.getMessage(); - if (msg != null && msg.contains("already in use")) { + if (isLockContention(t)) { LOG.info("orphan slot already locked, skipping: {} ({})", - slotPath, msg); + slotPath, t.getMessage()); outcome = DrainOutcome.LOCKED_BY_OTHER; return; } throw t; } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + logicalSlotLock = null; + } long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", @@ -575,7 +638,9 @@ public void run() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN); loop.start(); while (!stopRequestedOrInterrupted()) { @@ -714,12 +779,20 @@ public void run() { + "slot lock releases when it exits", slotPath); } } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + } // Don't let a later requestStop() unpark an unrelated task that // the pool's executor may have scheduled onto this same thread. runnerThread = null; } } + private static boolean isLockContention(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.contains("already in use"); + } + /** * Plug an observer for durable-ack-related events. {@code null} clears * any previously installed listener. See {@link BackgroundDrainerListener} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..597c2bc7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -24,10 +24,14 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import java.util.concurrent.locks.LockSupport; @@ -84,9 +88,9 @@ public final class CursorSendEngine implements QuietCloseable { private final SlotLock slotLock; // True when the constructor recovered an existing on-disk slot rather // than starting fresh. Diagnostic accessor for tests and observability; - // cursor frames are self-sufficient (every frame carries full schema + - // full symbol-dict delta), so producer-side schema reset on recovery - // is not required. + // every frame carries its full inline schema, so producer-side schema reset + // on recovery is not required (the symbol dictionary, which delta frames do + // NOT carry in full, is re-registered by an I/O-thread catch-up instead). private final boolean wasRecoveredFromDisk; // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a // ring recovered from disk, or -1 for fresh/memory rings and recovered @@ -96,6 +100,23 @@ public final class CursorSendEngine implements QuietCloseable { // covers them. Read by the sender's close-time drain to avoid waiting on // acks that cannot arrive. private long recoveredCommitBoundaryFsn = -1L; + // Highest symbol id any recovered delta frame references, or -1 for + // fresh/memory rings (and recovered rings with no symbol-bearing frame). A + // resuming producer seeds its dictionary baseline from the persisted + // .symbol-dict; if that dictionary was torn below this id by a host crash + // (the side-file is not fsync'd), the producer would re-use ids the surviving + // frames already define. seedGlobalDictionaryFromPersisted compares this + // against the recovered dictionary size to fail clean instead. Computed once + // in the constructor's recovery branch; -1 elsewhere. + private long recoveredMaxSymbolId = -1L; + // Highest deltaStart across the recovered COMMITTED frames; 0 when none carries a symbol + // dictionary. ZERO means every surviving frame is SELF-SUFFICIENT -- it re-registers its + // dictionary from id 0 -- so the slot replays with no dictionary at all and the send loop + // needs no catch-up. ABOVE zero means at least one frame is a true delta whose ids depend + // on registrations it does not itself carry, so the loop must seed its mirror (and ship a + // catch-up) before replaying. Both the full-dict-fallback discard below and the send + // loop's mirror seeding key off this. + private long recoveredMaxSymbolDeltaStart; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -110,6 +131,17 @@ public final class CursorSendEngine implements QuietCloseable { // in the constructor, closed by {@link #close()}. The segment manager // writes through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; + // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null} + // in memory mode and if open() failed). Enables delta-encoded SF frames: + // recovery / orphan-drain load it to re-register the dictionary on the fresh + // server before replaying non-self-sufficient frames. Opened in the + // constructor, closed by {@link #close()}. When null in disk mode the engine + // reports delta encoding as unavailable and the sender keeps full-dict frames. + private final PersistedSymbolDict persistedSymbolDict; + // Engine-owned output of the single ordered recovery walk. It is retained + // because both producer seeding and every recycled send loop need the same + // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. + private final RecoveredFrameAnalysis recoveredFrameAnalysis; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -137,9 +169,29 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, FilesFacade.INSTANCE); + } + + /** + * As {@link #CursorSendEngine(String, long, long, long)}, but with an explicit + * {@link FilesFacade} for the persisted symbol dictionary. + *

    + * The seam exists so a test can drive a dictionary I/O fault -- a short write from + * a full disk or an exhausted quota -- through the REAL producer path + * ({@code flush()} -> the write-ahead persist), and assert it surfaces as a + * {@code LineSenderException} like every other flush-path failure rather than as a + * raw {@code IllegalStateException} that would sail past every caller's + * {@code catch (LineSenderException)}. Nothing else could reach that translation: + * {@code PersistedSymbolDict} has facade-aware overloads, but the engine used to + * call only the {@code FilesFacade.INSTANCE} ones, so no fault could be injected + * from outside. + */ + @TestOnly + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, FilesFacade dictFf) { this(sfDir, segmentSizeBytes, new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes), - true, appendDeadlineNanos); + true, appendDeadlineNanos, dictFf); } /** @@ -148,11 +200,12 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, * ownership of the manager. Uses the default append deadline. */ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager) { - this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS); + this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS, + FilesFacade.INSTANCE); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, - boolean ownsManager, long appendDeadlineNanos) { + boolean ownsManager, long appendDeadlineNanos, FilesFacade dictFf) { // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments // live in malloc'd native memory. @@ -199,7 +252,17 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // reference instead of orphaning the mmap'd segments + fds. SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; + PersistedSymbolDict persistedDictInProgress = null; + RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { + // v2 segment payloads may depend on a persisted symbol-dictionary + // prefix. Install the rollback barrier before recovery or any new + // append so a v1-only client can never skip the v2 files, treat the + // slot as empty, and silently restart at FSN 0. Current recovery + // recognizes and skips the reserved guard filenames. + if (!memoryMode) { + SegmentRing.installLegacyReaderBarrier(sfDir); + } // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs @@ -257,6 +320,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. watermarkInProgress = AckWatermark.open(sfDir); + // Load the persisted symbol dictionary so delta-encoded frames + // in this recovered slot can be re-registered on the fresh + // server before replay. Null on open failure -> delta disabled. + persistedDictInProgress = PersistedSymbolDict.open(dictFf, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -276,6 +343,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (seed >= 0) { recovered.acknowledge(seed); } + // Fold the whole recovered ring once. The result checkpoints all + // running metadata and raw symbol bytes at each commit-bearing + // frame, so its final snapshot excludes an orphan deferred tail + // without requiring a second bounded scan. + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery( + persistedDictInProgress == null ? 0 : persistedDictInProgress.size()); // Locate the last commit-bearing frame below a potentially // orphaned FLAG_DEFER_COMMIT tail. A producer that crashed (or // closed) mid-transaction leaves deferred frames with no @@ -285,12 +358,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // drainOnClose), and (b) replaying them into a NEW session's // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. - this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE - ); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; LOG.warn("recovered SF log ends with {} deferred frame(s) whose transaction was never " @@ -300,6 +368,43 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } + // Highest symbol id the surviving COMMITTED frames reference. A + // resuming producer compares this against its recovered dictionary + // size (seedGlobalDictionaryFromPersisted) to detect a host-crash + // tear: if a committed frame references an id the (unsynced, torn) + // .symbol-dict no longer holds, resuming would re-use it. The walk is + // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred + // tail -- retired without ever being transmitted -- does not inflate + // this and over-reject an otherwise-recoverable slot. maxSymbolDeltaEnd + // returns 0 when no such frame carries a symbol, yielding -1 here. + // Computed before the I/O loop or producer append; single-threaded. + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + // Full-dict-fallback recovery. When the persisted .symbol-dict is a + // SUBSET of the ids the surviving frames reference + // (recoveredMaxSymbolId >= its size) YET every such frame is + // self-sufficient (maxSymbolDeltaStart == 0 -- a full-dict frame that + // re-registers its dictionary from id 0), the slot was written in + // full-dict fallback: the dictionary never opened when writing, so no + // side-file exists and this recovery opened a FRESH EMPTY one. Those + // frames replay with no dictionary, so discard the empty side-file and + // recover in full-dict mode -- isDeltaDictEnabled() then reports false + // and the producer + send loop both run full-dict, exactly as the slot + // was written. Without this the sender's seed-time guard would treat the + // empty dictionary as a host-crash tear and brick build(), even though + // the orphan drainer drains the same frames fine. A genuine torn DELTA + // dictionary keeps a frame with deltaStart > 0 (maxSymbolDeltaStart > 0) + // and is NOT discarded here: it still fails clean at seed time, since + // the ids its delta frames reference cannot be rebuilt without the lost + // dictionary. The recoveredMaxSymbolId >= size guard means this never + // fires for a slot whose dictionary is intact, nor for an empty slot + // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recoveredMaxSymbolDeltaStart == 0L) { + persistedDictInProgress.close(); + persistedDictInProgress = null; + } } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers @@ -309,6 +414,19 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); + // A fresh slot MUST start with an EMPTY symbol dictionary. + // Unlike the ack watermark above -- a discardable optimization a + // max() clamp protects -- the dictionary is load-bearing: a + // delta frame referencing an id missing from it is unrecoverable, + // and a STALE dictionary inherited here (the segments are gone, so + // the producer is NOT seeded from it) shifts the dense id->symbol + // mapping and silently misattributes symbols on the next + // reconnect. openClean() truncates any survivor to empty rather + // than trusting a best-effort delete that may have failed (e.g. a + // Windows share lock); if the clean open itself fails, + // persistedSymbolDict stays null and the sender falls back to full + // self-sufficient frames, which is also safe. + persistedDictInProgress = PersistedSymbolDict.openClean(dictFf, sfDir); } MmapSegment initial; String initialPath = null; @@ -333,10 +451,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man manager.start(); } manager.register(ringInProgress, sfDir, watermarkInProgress); - // All construction succeeded — commit the ring and - // watermark references. + // All construction succeeded — commit the ring, watermark and + // symbol-dictionary references. this.ring = ringInProgress; this.watermark = watermarkInProgress; + this.persistedSymbolDict = persistedDictInProgress; + this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -362,6 +482,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (persistedDictInProgress != null) { + try { + persistedDictInProgress.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysisInProgress != null) { + try { + recoveredFrameAnalysisInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -541,6 +673,18 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (persistedSymbolDict != null) { + try { + persistedSymbolDict.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysis != null) { + try { + recoveredFrameAnalysis.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -550,6 +694,11 @@ public synchronized void close() { AckWatermark.removeOrphan(sfDir); } catch (Throwable ignored) { } + try { + // Slot fully drained: the dictionary has no frames behind it. + PersistedSymbolDict.removeOrphan(sfDir); + } catch (Throwable ignored) { + } } } finally { if (slotLock != null) { @@ -562,6 +711,113 @@ public synchronized void close() { } } + /** + * Rebuilds, from the surviving frames' OWN delta sections, the symbols the upcoming + * replay will register ABOVE {@code baseline}, appending them to {@code out} in + * ascending id order. Returns the coverage the replay establishes (one past the highest + * id it registers), or {@code -1} when the frames have a genuine GAP above the baseline. + *

    + * The producer seeds its dictionary from the persisted {@code .symbol-dict} and THEN + * from this, so it is fed the same entries, in the same order, that the send loop's + * {@code accumulateSentDict} will feed its mirror from as those very frames replay. The + * producer's delta baseline and the loop's mirror coverage therefore land on the same + * number by construction, which is the invariant the torn-dictionary guard rests on. + *

    + * This is what makes a slot whose dictionary was torn -- or lost outright -- still + * recoverable when the surviving frames define the ids themselves (they carry the + * symbols in their own deltas; it is why the orphan drainer can drain such a slot). Only + * a real gap, where the ids were introduced by frames since acked and trimmed away, is + * unrecoverable -- and that is precisely when this returns -1. + *

    + * Bounded above by {@link #recoveredCommitBoundaryFsn} like {@link #recoveredMaxSymbolId}: + * frames past it are the aborted orphan-deferred tail, retired without ever being + * transmitted, so their ids never reach a server and must not inflate the baseline. + */ + public long collectReplaySymbolsAbove(int baseline, ObjList out) { + if (recoveredFrameAnalysis != null + && recoveredFrameAnalysis.baseline() == baseline) { + recoveredFrameAnalysis.appendDecodedSymbols(out); + return recoveredFrameAnalysis.coverage(); + } + if (ring == null) { + return baseline; + } + return ring.collectReplaySymbolsAbove( + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT, + QwpConstants.HEADER_SIZE, + // Walk from the LOWEST frame still on disk, not from ackedFsn+1. An acked frame + // is not trimmed the instant it is acked -- trim drops whole SEALED segments -- + // so the symbols it registered are usually still sitting right there, and they + // are exactly the ids the replay set's deltas start above. Skipping them threw + // away the only surviving copy of those symbols and condemned a slot that had + // everything it needed. A slot whose registering frames really HAVE been + // trimmed still reports the gap, because then no frame on disk holds them. + 0L, + recoveredCommitBoundaryFsn, + baseline, + out + ); + } + + /** + * Decodes the cached recovery suffix directly into the producer's global + * dictionary. Recovery always builds the analysis with the persisted + * prefix size as its baseline, so no intermediate cardinality-sized list is + * needed on the production path. + */ + public long addRecoveredSymbolsTo(int baseline, GlobalSymbolDictionary target) { + if (recoveredFrameAnalysis == null) { + return baseline; + } + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + long coverage = analysis.coverage(); + if (coverage >= 0L) { + analysis.addDecodedSymbolsTo(target); + } + return coverage; + } + + long recoveredSymbolCoverage(int baseline) { + return checkedRecoveryAnalysis(baseline).coverage(); + } + + int recoveredSymbolSuffixCount(int baseline) { + return checkedRecoveryAnalysis(baseline).rawCount(); + } + + int recoveredSymbolSuffixLen(int baseline) { + return checkedRecoveryAnalysis(baseline).rawLen(); + } + + void copyRecoveredSymbolSuffix(int baseline, long target) { + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + int len = analysis.rawLen(); + if (len > 0) { + io.questdb.client.std.Unsafe.getUnsafe().copyMemory(analysis.rawAddr(), target, len); + } + } + + @TestOnly + public long recoveryFramesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited(); + } + + @TestOnly + public long recoverySymbolEntriesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.symbolEntriesVisited(); + } + + private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) { + if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) { + throw new IllegalStateException("recovery symbol baseline mismatch [expected=" + + (recoveredFrameAnalysis == null ? "none" : recoveredFrameAnalysis.baseline()) + + ", actual=" + baseline + ']'); + } + return recoveredFrameAnalysis; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ @@ -576,6 +832,15 @@ public MmapSegment firstSealed() { return ring.firstSealed(); } + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Number of times {@link #appendBlocking} hit * {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and @@ -586,6 +851,18 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } + /** + * Whether the sender may delta-encode symbol dictionaries on this engine. + * Always true in memory mode (the send loop keeps an in-process catch-up + * mirror). In disk mode it requires the persisted dictionary to have opened, + * since delta frames are not self-sufficient and recovery / orphan-drain must + * be able to rebuild the dictionary from disk. When false in disk mode the + * sender falls back to full self-sufficient frames. + */ + public boolean isDeltaDictEnabled() { + return sfDir == null || persistedSymbolDict != null; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ @@ -650,6 +927,30 @@ public long recoveredCommitBoundaryFsn() { return recoveredCommitBoundaryFsn; } + /** + * Highest symbol id any recovered delta frame references, or {@code -1} for + * fresh/memory rings (and recovered rings with no symbol-bearing frame). A + * resuming producer compares this against its recovered dictionary size to + * detect a host-crash tear of the persisted {@code .symbol-dict}. + */ + public long recoveredMaxSymbolId() { + return recoveredMaxSymbolId; + } + + /** + * Highest {@code deltaStart} across the recovered committed frames; {@code 0} when every + * surviving frame is self-sufficient (or none carries a dictionary at all). + *

    + * The send loop uses this to decide whether it needs a catch-up: at zero, every frame + * re-registers its dictionary from id 0 as it replays, so seeding the mirror -- and + * shipping a catch-up frame off it -- would be pure redundancy. Above zero, at least one + * frame's delta starts above ids it does not itself carry, so the mirror must hold those + * ids before the replay begins or the server null-pads the hole. + */ + public long recoveredMaxSymbolDeltaStart() { + return recoveredMaxSymbolDeltaStart; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..e9af2d5d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -31,15 +31,19 @@ import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketUpgradeException; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.str.Utf8s; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; @@ -49,6 +53,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -65,8 +70,10 @@ *

  6. On wire failure, runs the configured reconnect policy: capped * exponential backoff with jitter, retried indefinitely (Invariant B -- * a store-and-forward drainer never gives up on a wall-clock budget), - * with only auth-style failures (401/403/non-101 upgrade reject) treated - * as terminal. On reconnect success, repositions the cursor at + * with endpoint-policy failures terminal only for an initial foreground + * connect or an orphan drainer. A previously-live foreground sender keeps + * retrying so credential and rolling-capability changes remain contained + * by store-and-forward. On reconnect success, repositions the cursor at * {@code ackedFsn+1} and replays.
    7. *
* No locks on the steady-state path. The producer thread (user) writes @@ -129,11 +136,75 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code LineSenderBuilder.poisonMinEscalationWindowMillis(long)}. */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; + /** + * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must + * persist before an orphan drainer may latch a terminal, even once + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as + * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a + * strike count measures "how many times did we look", not "how long has this been + * true", and only the latter distinguishes a permanent cluster capability gap from + * a node that is briefly away. + *

+ * It matters here because the cap gap's trigger IS a failover. With the reconnect + * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap + * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling + * restart of the larger-cap node. A count-only budget could therefore quarantine an + * otherwise drainable orphan slot during a routine cluster operation. Requiring the + * dwell as well means the orphan terminal needs BOTH "we have looked many times" AND + * "it has stayed true for a long time". Foreground senders never apply this terminal + * policy: they retry indefinitely until a larger-cap node returns. + *

+ * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, + * this codebase's existing notion of how long a transient outage may plausibly + * last. Configurable per sender via the + * {@code catchup_cap_gap_min_escalation_window_millis} connect-string key or + * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} + * restores count-only escalation for orphan drainers. + */ + public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts + // -- catch-ups that reached a fresh server and found a single dictionary entry + // too large for its advertised batch cap -- may occur consecutively before an + // orphan drainer latches a terminal. This is a + // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), + // the connect-time analog of the orphan drainer's durable-ack capability gap + // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips + // it -- an entry that fit its data frame under a cap always fits its bare catch-up + // frame under that same cap -- so it only affects a heterogeneous / rolling-cap + // cluster, where a failover to a smaller-cap node can hit it for an entry an + // earlier node accepted. A foreground sender retries forever. An orphan drainer + // rides out the transient window until a larger-cap node returns; only a persistent + // gap (this many consecutive cap gaps with no successful catch-up or unrelated + // reconnect state in between) latches. + // + // Budget accounting (satisfies "a transient must never burn the terminal + // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a + // node is reached and an entry is oversized. A successful catch-up ends the + // episode, as does any unrelated reconnect state (connect refusal, catch-up send + // failure, upgrade/role rejection): otherwise its downtime would count toward the + // wall-clock dwell even though no cap gap was observed. The cap-gap exception + // itself does NOT reset the episode, so consecutive small-cap nodes still prove a + // persistent cluster capability gap and can exhaust the orphan policy. + private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; + // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror + // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even + // this needs ~200M+ distinct symbols on a single connection, far past any real + // workload. The guard exists so that pathological growth fails loudly instead + // of overflowing the int capacity math into a heap-corrupting copyMemory. + private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // Test seam: when true, recovery mirror seeding throws before growing, + // simulating the native realloc OOM (or MAX_SENT_DICT_BYTES ceiling) that + // ensureSentDictCapacity can raise while the constructor seeds the recovery + // mirror. Lets a test prove the constructor frees the already-malloc'd mirror on + // such a throw rather than leaking it. Production never sets it; volatile only so + // a test thread's write is visible to the loop under test. + @TestOnly + static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -151,6 +222,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); + // Pre-converted to nanos. Consulted only by the orphan terminal policy. Zero disables + // the dwell entirely (count-only escalation at MAX_CATCHUP_CAP_GAP_ATTEMPTS); the + // user-facing 5-minute default is applied at the config layer. + private final long catchUpCapGapMinEscalationWindowNanos; + private final CatchUpCapGapPolicy catchUpCapGapPolicy; + private final ReconnectPolicy reconnectPolicy; private final CursorSendEngine engine; private final long parkNanos; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only @@ -200,6 +277,74 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); + // Delta symbol dictionary catch-up state (see swapClient). + // ALWAYS active -- in memory mode, in disk mode, and (critically) even when the + // per-slot persisted dictionary failed to open. sentDictCount is this loop's model + // of how many ids the CURRENT server has been told about, and the torn-dict guard + // in trySendOne reads it, so it must track the wire in every mode; gating it on + // engine.isDeltaDictEnabled() is what once froze it at 0 and terminal'd a slot + // whose frames replay perfectly from id 0 (see the constructor). On a recovered / + // orphan-drained slot the constructor SEEDS sentDict* from the persisted + // dictionary when there is one; otherwise the mirror starts empty and grows from + // the frames themselves. The loop mirrors, in sentDict*, every symbol it has ever + // sent -- the concatenated [len varint][utf8] bytes in global-id order + // (sentDictBytes*) plus the count (sentDictCount) -- so that on reconnect it can + // re-register the whole dictionary on the fresh server (which discards its + // dictionary on every disconnect) before replaying frames whose deltas start above + // id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed for + // the connection's lifetime (a reconnect may need the whole dictionary at any + // moment), so it cannot be dropped; it is an intentional cost of the feature. + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + // False only while an orphan-drainer loop borrows the persisted prefix. + // Any growth first performs a copy-on-write; only owned buffers are freed. + private boolean sentDictBytesOwned; + private int sentDictCount; + // Reusable staging frame for dictionary catch-up chunks. A reconnect may split a + // large dictionary into many chunks, and every later reconnect repeats that split; + // retaining one growable loop-owned buffer turns the old malloc/free-per-chunk path + // into amortized growth only. I/O-thread-owned, freed beside sentDictBytesAddr. + private long catchUpFrameAddr; + private int catchUpFrameCapacity; + private int catchUpFrameGrowthCount; + // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry + // too large for its batch cap -- with no intervening successful catch-up or unrelated + // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). + // Foreground retries never increment it. I/O-thread-only. + private int catchUpCapGapAttempts; + // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. + // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a + // successful catch-up or an unrelated reconnect state, so only an uninterrupted run + // of cap-gap observations contributes wall-clock dwell. Anchoring at the FIRST gap + // (not refreshed per attempt) lets consecutive small-cap nodes satisfy the dwell; + // resetting after transport/role states prevents their downtime from doing so. + // I/O-thread-only, like catchUpCapGapAttempts. + // + // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test + // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant + // is only meaningful as a difference: its origin is arbitrary and the spec permits + // negative values, so no state may ride on this field's sign. + private long catchUpCapGapFirstNanos = -1L; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up consumes + // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head + // frame was sent": onClose's poison-strike gate and handleServerRejection's + // pre-send gate key off THIS instead. Without it, a transient outage AFTER the + // catch-up but BEFORE the first data frame (a flapping LB/middlebox that accepts + // the upgrade + catch-up then closes) would be mistaken for a deterministic + // head-frame rejection and escalate to a PROTOCOL_VIOLATION terminal -- breaking + // the store-and-forward "retry a transient outage forever" contract. Reset per + // connection in setWireBaselineWithCatchUp; set in trySendOne after a successful + // send. + private boolean dataFrameSentThisConnection; + // End position (native address) written by the last readVarintAt() call. + private long varintEnd; private WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's @@ -228,9 +373,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a // successful connect attempt in any mode). Once true, stays true. Used to - // distinguish a "never reached the server" terminal failure (looks like a - // config typo or firewall block) from "lost connection after we were - // up" (looks transient). + // distinguish an initial endpoint-policy failure (fail fast) from a + // post-start failure that a foreground sender must contain and retry. private volatile boolean hasEverConnected; private volatile Thread ioThread; // Typed marker for a durable-ack CAPABILITY-GAP terminal: set (before the @@ -239,8 +383,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // QwpDurableAckMismatchException. The orphan drainer consults it to route // a mid-drain capability gap into its budgeted settle-retry // (BackgroundDrainer.connectWithDurableAckRetry) instead of quarantining - // the slot on the first sweep; the foreground sender ignores it and keeps - // its spec'd loud-fail (sf-client.md section 8.1). Write-once alongside + // the slot on the first sweep. Foreground reconnects never set this marker; + // they keep retrying after a successful initial connection. Write-once alongside // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; @@ -370,9 +514,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code client} may be {@code null} only if {@code reconnectFactory} * is non-null — this is the async-initial-connect path: the I/O thread * runs the same retry loop on its first iteration to obtain a live - * client, and a terminal failure (auth/upgrade reject) is delivered - * through the dispatcher rather than thrown to the constructor's - * caller; plain connect failures are retried indefinitely + * client, and an initial terminal failure (auth/upgrade reject or durable-ack + * mismatch) is delivered through the dispatcher rather than thrown to the + * constructor's caller; plain connect failures are retried indefinitely * (Invariant B: no wall-clock budget give-up). */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, @@ -454,13 +598,45 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } /** - * Master constructor — also accepts the poison-frame detector threshold + * Twelve-arg overload — omits the symbol-dict cap-gap escalation dwell, which + * defaults to {@code 0}. This and the thirteen-arg overload use the foreground-safe + * retry-forever policy; orphan drainers opt into count-and-dwell escalation through + * the policy-aware master constructor. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, 0L); + } + + /** + * Thirteen-arg overload. Catch-up cap gaps are retried indefinitely, which is the + * required policy for a foreground store-and-forward sender. Orphan drainers use the + * policy-aware overload below to opt into bounded terminal escalation. + *

+ * Also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the - * loop escalates to a typed terminal. Must be {@code >= 1}. The final - * argument is the minimum wall-clock dwell (millis) the suspect must stay - * poisoned before escalation ({@code poison_min_escalation_window_millis}; - * {@code >= 0}, where 0 = legacy immediate escalation at the threshold). + * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum + * wall-clock dwell (millis) the suspect must stay poisoned before escalation + * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = legacy + * immediate escalation at the threshold). + *

+ * The final argument is the analogous dwell for the symbol-dict catch-up cap gap. + * It is retained here for source and binary compatibility but is consulted only when + * the policy-aware overload selects + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET}. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -471,7 +647,33 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis) { + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy.RETRY_FOREVER); + } + + /** + * Compatibility policy-aware constructor. New production call sites should + * use the {@link ReconnectPolicy} overload below, which names the foreground + * versus orphan ownership distinction directly. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy catchUpCapGapPolicy) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -480,6 +682,24 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException( "poisonMinEscalationWindowMillis must be >= 0: " + poisonMinEscalationWindowMillis); } + if (catchUpCapGapMinEscalationWindowMillis < 0) { + throw new IllegalArgumentException( + "catchUpCapGapMinEscalationWindowMillis must be >= 0: " + + catchUpCapGapMinEscalationWindowMillis); + } + // TimeUnit conversion saturates at Long.MAX_VALUE. A raw multiply + // wraps large valid millisecond values negative, which makes the + // elapsed-time gate appear satisfied as soon as the strike threshold + // is reached and can quarantine a recoverable orphan slot prematurely. + this.catchUpCapGapMinEscalationWindowNanos = + TimeUnit.MILLISECONDS.toNanos(catchUpCapGapMinEscalationWindowMillis); + if (catchUpCapGapPolicy == null) { + throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); + } + this.catchUpCapGapPolicy = catchUpCapGapPolicy; + this.reconnectPolicy = catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER + ? ReconnectPolicy.FOREGROUND + : ReconnectPolicy.ORPHAN; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -489,6 +709,102 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, + // so seed it from the slot's persisted dictionary. That way the very first + // connection re-registers the whole dictionary (via a catch-up frame) + // before replaying the recovered delta frames. + // + // Deliberately NOT gated on engine.isDeltaDictEnabled(). The mirror, the + // catch-up and the replay guard together are this loop's model of what the + // SERVER's dictionary holds, and that model must track every mode. Gating + // them on that flag is what once bricked a recoverable slot: an unopenable + // .symbol-dict reports isDeltaDictEnabled()=false, yet the frames already on + // disk are still DELTA frames (deltaStart 0,1,2,...). With the mirror gated + // off, sentDictCount froze at 0 while the ungated guard kept comparing + // against it, so the frame at deltaStart=1 tripped a terminal -- even though + // replaying the whole sequence from id 0 rebuilds the dictionary on the + // server contiguously and drains perfectly. isDeltaDictEnabled() decides + // what the PRODUCER emits and persists; it must never decide what this loop + // tracks. When the dictionary could not be opened, pd is null, the mirror + // simply starts empty and grows from the frames themselves. + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + int persistedPrefixLen = 0; + if (pd != null && pd.size() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + // Seed by reference. The foreground loop takes ownership after + // construction succeeds; orphan-drainer loops keep borrowing because + // one engine can create several sessions during capability-gap + // recycling. A borrowed mirror performs copy-on-write if a recovered + // frame suffix must extend it. + persistedPrefixLen = len; + sentDictBytesAddr = pd.loadedEntriesAddr(); + sentDictBytesCapacity = len; + sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold. A recovered slot + // always has loadedEntriesLen > 0 when size > 0, so this is the + // same result -- it just makes the coupling explicit. + sentDictCount = pd.size(); + } + } + // ...and then from the surviving frames' own delta sections, exactly as the producer + // seeds its dictionary. The mirror is the loop's model of what the SERVER holds and it + // is what the catch-up frame ships, so when it stops at the persisted prefix while the + // producer's baseline runs past it, the loop condemns (deltaStart > sentDictCount) a + // slot the producer just seeded successfully. Same two sources, same order, so the two + // land on the same number by construction. + // ...but ONLY when a surviving frame actually depends on ids it does not carry + // (recoveredMaxSymbolDeltaStart > 0). At zero every frame is self-sufficient and + // re-registers its dictionary from id 0 as it replays, so seeding the mirror would buy + // nothing and cost a catch-up frame on every connection -- full-dict slots must stay + // catch-up-free. + if (engine.recoveredMaxSymbolDeltaStart() > 0L) { + // The prefix seed above may already have malloc'd the mirror, and + // ensureSentDictCapacity can still throw here (a native realloc OOM, or + // the MAX_SENT_DICT_BYTES ceiling). Such a throw + // propagates out of the constructor, so the half-built loop is never + // assigned to a reference -- neither ensureConnected's catch nor + // BackgroundDrainer's finally can close() it -- and the mirror would leak. + // Free it on any throw so the constructor leaves nothing behind, mirroring + // the loopNeverRan free in close() / ioLoop's exit. + try { + int baseline = sentDictCount; + if (engine.recoveredSymbolCoverage(baseline) >= 0L) { + int suffixLen = engine.recoveredSymbolSuffixLen(baseline); + int suffixCount = engine.recoveredSymbolSuffixCount(baseline); + if (suffixLen > 0) { + if (forceMirrorSeedFailureForTest) { + throw new LineSenderException( + "simulated mirror seed allocation failure (test only)"); + } + ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); + engine.copyRecoveredSymbolSuffix( + baseline, sentDictBytesAddr + sentDictBytesLen); + sentDictBytesLen += suffixLen; + sentDictCount += suffixCount; + } + } + } catch (Throwable t) { + releaseSentDictBytes(); + throw t; + } + } + // QwpWebSocketSender seeds the producer dictionary before constructing + // its one foreground loop, so that loop can own the recovered prefix and + // eliminate the engine-side duplicate. BackgroundDrainer must retain the + // prefix for later recycled loops and therefore keeps borrowing it. + if (persistedPrefixLen > 0 && catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { + long persistedPrefixAddr = pd.takeLoadedEntries(); + if (sentDictBytesOwned) { + // Copy-on-grow already produced the combined mirror. Retire the + // no-longer-needed persisted prefix after successful construction. + Unsafe.free(persistedPrefixAddr, persistedPrefixLen, MemoryTag.NATIVE_DEFAULT); + } else { + assert persistedPrefixAddr == sentDictBytesAddr; + sentDictBytesOwned = true; + } + } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; @@ -519,6 +835,41 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } + /** + * Policy-aware master constructor. A foreground sender fails fast while + * establishing its first connection, then retries endpoint-policy failures + * indefinitely after it has been live. An orphan drainer returns such failures + * to its owner so the slot can follow its settle/quarantine policy. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + ReconnectPolicy reconnectPolicy) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + catchUpPolicyFor(reconnectPolicy)); + } + + private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) { + if (reconnectPolicy == null) { + throw new IllegalArgumentException("reconnectPolicy must be non-null"); + } + return reconnectPolicy == ReconnectPolicy.FOREGROUND + ? CatchUpCapGapPolicy.RETRY_FOREVER + : CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET; + } + /** * Maps a server status byte to a {@link SenderError.Category}. Exposed for unit tests. */ @@ -754,6 +1105,12 @@ public synchronized void close() { // after — the latch await is only skipped when the loop never ran. running = false; Thread t = ioThread; + // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets + // freed on ioLoop's exit path. When t == null the loop never ran (start() + // was never called, or t.start() failed before committing ioThread), so + // that free never happens and a seeded (recovery / orphan-drain) mirror + // would leak. Capture that here; free it below, after client teardown. + boolean loopNeverRan = t == null; if (t != null) { LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. @@ -812,6 +1169,17 @@ public synchronized void close() { } client = null; } + // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never + // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and + // on the failed-stop path (interrupted latch await, ioThread left set) the + // thread may still be mid-send, so touching the mirror here would race. + // A duplicate close observes sentDictBytesAddr == 0 and skips. + if (loopNeverRan && sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + if (loopNeverRan) { + freeCatchUpFrameBuffer(); + } } /** @@ -996,7 +1364,28 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // A recovered sender re-registers its dictionary with a catch-up on + // the very first connect. Here that runs on the CALLER thread (sync + // start), so we must NOT let it drive connectLoop -- that would block + // Sender construction forever on a transient outage. Drop the dead + // client instead: the I/O thread then reconnects via + // attemptInitialConnect -> swapClient and re-sends the catch-up off + // this thread. If the failure was already terminal (recordFatal set + // running=false, e.g. an entry too large for the batch cap), the I/O + // thread simply winds down and checkError() surfaces it. + WebSocketClient dead = client; + client = null; + if (dead != null) { + try { + dead.close(); + } catch (Throwable ignored) { + // best-effort + } + } + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { @@ -1104,6 +1493,15 @@ private void clearDurableAckTracking() { lastFrameOrPingNanos = 0L; } + private static boolean isCatchUpCapGap(Throwable t) { + return t instanceof CatchUpSendException && ((CatchUpSendException) t).capGap; + } + + private void resetCatchUpCapGapEpisode() { + catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; + } + /** * Shared per-outage retry loop. Used by {@link #fail(Throwable)} for * mid-flight wire failures (phase="reconnect") and by @@ -1118,16 +1516,25 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + // A wire/role state that interrupted an orphan cap-gap run is not evidence that + // the cluster's batch cap stayed incompatible during the outage. Start a fresh + // episode; the cap-gap exception itself is the one reconnect cause that preserves + // the existing attempt count and dwell anchor. + if (!isCatchUpCapGap(initial)) { + resetCatchUpCapGapEpisode(); + } LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); - // INVARIANT B: a store-and-forward drainer must NEVER terminate on a + // INVARIANT B: a store-and-forward loop must NEVER terminate on a // wall-clock reconnect budget. A replica-only / all-endpoints-replica // window is TRANSIENT -- a replica gets promoted, a primary reappears -- // so this background loop retries for as long as it is running, backing - // off between attempts. The ONLY terminal conditions are a genuinely - // non-retriable upgrade (auth / non-421 upgrade / durable-ack capability - // gap), which return directly below, or the sender being stopped. SF + // off between attempts. Endpoint-policy failures (auth / non-421 + // upgrade / durable-ack capability gap) are terminal only for orphan + // drainers and a foreground sender's first connection. Once a foreground + // sender has connected, those states are retried so a credential or cluster + // capability rotation cannot stop its producer. SF // exhaustion is surfaced to the PRODUCER as append backpressure, never // here. reconnect_max_duration_millis is intentionally NOT consulted: it // bounds only the blocking (non-lazy) initial connect in @@ -1187,66 +1594,82 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, fsnAtZero); return; } + // A null factory result is an unsuccessful connect state, not a cap-gap + // observation. Do not let the time spent retrying it satisfy orphan dwell. + resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { - // Terminal across all configured endpoints per spec sf-client.md - // section 13.3: auth (401/403) bypasses reconnect and surfaces as - // SECURITY_ERROR. WebSocketUpgradeException reaching here is always - // non-421: QwpUpgradeFailures.classify upstream converts a - // 421-with-X-QuestDB-Role to QwpIngressRoleRejectedException, and a - // 421 without that header walks the transport-error path in - // buildAndConnect and lands as a LineSenderException, falling into - // the Throwable branch below. - LOG.error("terminal upgrade error during {} -- won't retry: {}", - phase, e.getMessage()); - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.SECURITY_ERROR, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "ws-upgrade-failed: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + if (endpointPolicyFailureIsTerminal()) { + // Orphans return control to their quarantine owner; a + // foreground sender that never connected still fails fast. + // WebSocketUpgradeException reaching here is always non-421: + // role rejects are classified into the transient branch below. + LOG.error("terminal upgrade error during {} -- won't retry: {}", + phase, e.getMessage()); + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.SECURITY_ERROR, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "ws-upgrade-failed: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground auth/upgrade policy rejected the connection; " + + "retrying because this sender was previously live -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpDurableAckMismatchException e) { - // Per spec sf-client.md section 8.1: the client opted into durable - // ack but the cluster cannot honour it. Loud fail at connect rather - // than silently waiting for ack frames that will never arrive. - // Classified as PROTOCOL_VIOLATION (config/capability mismatch), - // not SECURITY_ERROR -- this is not an auth failure. - LOG.error("durable-ack mismatch during {} -- won't retry: {}", - phase, e.getMessage()); - if (terminalError == null) { - // Mirror recordFatal's first-writer-wins latch: only the - // sweep that owns the terminal may mark the gap, and the - // marker must be visible before the terminalError volatile - // write that checkError() keys on. - capabilityGapTerminal = e; + if (endpointPolicyFailureIsTerminal()) { + // Orphans hand a capability gap back to BackgroundDrainer's + // settle budget. An initial foreground connect remains loud. + LOG.error("durable-ack mismatch during {} -- won't retry: {}", + phase, e.getMessage()); + if (terminalError == null) { + // Publish the marker before terminalError, which is the + // volatile first-writer-wins latch observed by the owner. + capabilityGapTerminal = e; + } + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "durable-ack-mismatch: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground durable-ack capability is temporarily " + + "unavailable; retrying because this sender was previously live -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; } - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.PROTOCOL_VIOLATION, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "durable-ack-mismatch: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; } catch (QwpRoleMismatchException | QwpIngressRoleRejectedException e) { // Role mismatch: every reachable endpoint role-rejected the // upgrade -- right now they are all replicas / primary-catchup. @@ -1263,6 +1686,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // endpoint, forever. Growing to reconnectMaxBackoffMillis // mirrors the orphan drainer's role-reject path and honours the // documented capped-exponential-backoff contract. + resetCatchUpCapGapEpisode(); lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1286,6 +1710,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(e); throw (Error) e; } + if (!isCatchUpCapGap(e)) { + resetCatchUpCapGapEpisode(); + } lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1331,6 +1758,10 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, lastMsg); } + private boolean endpointPolicyFailureIsTerminal() { + return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; + } + /** * Send {@code err} to the async-delivery dispatcher if one is configured. * Producer-side typed throw (TERMINAL) goes through {@code recordFatal} + @@ -1669,6 +2100,12 @@ private void ioLoop() { // best-effort } } + // The symbol-dict mirror is I/O-thread-owned; free it here, on the + // owning thread's exit path, after the last send that could touch it. + if (sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + freeCatchUpFrameBuffer(); shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -1691,8 +2128,11 @@ private void ioLoop() { /** * Walk the engine's segments to find the one containing {@code targetFsn}, * and set {@code sendOffset} to the byte offset of that frame within it. - * This is called at startup and after every reconnect, after fsnAtZero has - * already been reset to {@code targetFsn} and nextWireSeq to 0. + * This is called at startup and after every reconnect, once + * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline + * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq} + * past the catch-up frames it emitted. This method only positions the byte + * cursor at {@code targetFsn}; it does not touch the wire mapping. *

* If {@code targetFsn} is already published, the method positions the byte * cursor exactly at that frame. If {@code targetFsn} is not published yet, @@ -1849,8 +2289,6 @@ private void swapClient(WebSocketClient newClient) { // past the tail instead of replaying into it. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; // Snapshot publishedFsn at swap time — frames at FSN ≤ this value // were already on the wire before the drop and will be replayed. // trySendOne resets replayTargetFsn to -1 once we cross the boundary. @@ -1862,9 +2300,490 @@ private void swapClient(WebSocketClient newClient) { // carrying stale state across the wire boundary would either // double-trim or starve the queue. clearDurableAckTracking(); + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } + /** + * Sets the wire-sequence baseline for a fresh connection and, when the symbol + * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first + * so the fresh server (whose dictionary starts empty) can resolve the + * non-self-sufficient delta frames that replay next. + *

+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just + * below {@code replayStart} (a harmless re-ack); real replay frames then follow + * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode), + * or before a client exists (async initial connect), keep the plain 1:1 + * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the + * first real connection via swapClient. + */ + private void setWireBaselineWithCatchUp(long replayStart) { + // Fresh connection: no data frame has been sent on it yet. Reset before the + // catch-up (which sends only dictionary frames) so onClose / + // handleServerRejection can tell "only the catch-up went out" from "the + // head data frame went out". + dataFrameSentThisConnection = false; + // Not gated on isDeltaDictEnabled() -- see the constructor. The mirror is + // non-empty exactly when this loop has already put symbols on a wire, and a + // FRESH server holds none of them, so it must be re-registered before any + // frame that back-references an id. In full-dict mode the mirror is non-empty + // too and the catch-up is then redundant (the next frame re-registers from id + // 0 anyway) but harmless: the server overwrites identical ids with identical + // symbols. Redundant-but-uniform beats a mode flag that can go false while the + // frames on disk are still deltas. + if (client != null && sentDictCount > 0) { + this.nextWireSeq = 0L; + // The catch-up may span several frames when the dictionary exceeds the + // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps + // to an already-acked FSN, so the first real frame still lands on + // replayStart. + int catchUpFrames = sendDictCatchUp(); + this.fsnAtZero = replayStart - catchUpFrames; + } else { + this.fsnAtZero = replayStart; + this.nextWireSeq = 0L; + } + } + + /** + * Returns the symbol-dictionary delta start id of a frame, or -1 when the + * frame carries no delta section. Used by the pre-send torn-dictionary guard. + */ + private int frameDeltaStart(long payloadAddr, int payloadLen) { + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return -1; + } + return (int) readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen); + } + + // True only for a well-formed QWP frame this encoder produced that carries a + // delta symbol-dict section. The magic check keeps the dict logic from + // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose + // bytes happen to set the delta flag. + private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + } + + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode, for a frame the + * pre-send guard already classified as a delta frame. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + * @param deltaStart the frame's delta start id, already decoded by the + * pre-send guard ({@link #frameDeltaStart}) -- passed in so + * the magic/flags and start-id varint are not re-parsed + */ + private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) { + long limit = payloadAddr + payloadLen; + // deltaStart is known (the guard decoded it); locate deltaCount just past + // its canonical LEB128 encoding rather than re-reading the header and the + // start-id varint. + long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); + long deltaCount = readVarintAt(p, limit); + p = varintEnd; + // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this + // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the + // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases: + // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard + // rejects it before send, so it never reaches here; bail defensively + // rather than accumulate past a hole. + // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold -- + // nothing new. + // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail. + // deltaStart == sentDictCount is the steady-state case (skip == 0). + // Handling the partial overlap explicitly -- rather than dropping the whole + // frame whenever deltaStart != sentDictCount -- keeps the mirror complete + // even if a future producer ever emits a delta that overlaps the tip; + // silently dropping the new tail would leave the reconnect catch-up + // incomplete and shift server-side ids. + long deltaEnd = deltaStart + deltaCount; + if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { + return; + } + // Walk past the already-held prefix [deltaStart, sentDictCount), then copy + // the new tail [sentDictCount, deltaEnd). + int skip = sentDictCount - deltaStart; + for (int i = 0; i < skip; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + return; // malformed -- bail rather than corrupt the mirror + } + } + long regionStart = p; + long newCount = deltaEnd - sentDictCount; + for (long i = 0; i < newCount; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + // Malformed -- never happens for frames we encoded; bail rather + // than corrupt the mirror. + return; + } + } + int regionBytes = (int) (p - regionStart); + // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on + // a very-high-cardinality lifetime connection; ensureSentDictCapacity then + // fails loudly rather than overflowing to a negative int (which would make + // the capacity check pass and copyMemory scribble past the buffer). + ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); + Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); + sentDictBytesLen += regionBytes; + sentDictCount += (int) newCount; + } + + /** + * Appends one symbol to the sent-dictionary mirror in wire form + * ({@code [len varint][utf8]}), advancing {@code sentDictCount}. Seeds the mirror at + * construction; {@link #accumulateSentDict} extends it from live frames thereafter. + */ + private void appendSymbolToMirror(CharSequence symbol) { + if (forceMirrorSeedFailureForTest) { + // Simulate a native realloc OOM on the seed path (see the field). + throw new LineSenderException("simulated mirror seed allocation failure (test only)"); + } + int utf8Len = Utf8s.utf8Bytes(symbol); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; + ensureSentDictCapacity((long) sentDictBytesLen + wireLen); + long p = NativeBufferWriter.writeVarint(sentDictBytesAddr + sentDictBytesLen, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + sentDictBytesLen += wireLen; + sentDictCount++; + } + + private void ensureSentDictCapacity(long required) { + if (sentDictBytesCapacity >= required) { + return; + } + if (required > MAX_SENT_DICT_BYTES) { + // Latch a terminal, do NOT just throw: accumulateSentDict runs AFTER + // the frame's sendBinary, so a bare throw unwinds to ioLoop -> fail() + // -> connectLoop, which (running still true) reconnects and replays the + // same frame, which re-overflows the never-shrinking mirror -- an + // unbounded reconnect livelock rather than the "fails loudly" the + // MAX_SENT_DICT_BYTES ceiling promises. recordFatal flips running=false + // so connectLoop's !running guard winds the loop down and checkError() + // surfaces the terminal, matching sendCatchUpChunk's guard for the same + // ceiling. The throw still unwinds past the pending copyMemory. + LineSenderException err = new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw err; + } + // Grow in long to avoid the capacity*2 int overflow (negative) that would + // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. + long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required)); + if (newCap > MAX_SENT_DICT_BYTES) { + newCap = MAX_SENT_DICT_BYTES; + } + if (sentDictBytesOwned) { + sentDictBytesAddr = Unsafe.realloc( + sentDictBytesAddr, + sentDictBytesCapacity, + (int) newCap, + MemoryTag.NATIVE_DEFAULT); + } else { + long newAddr = Unsafe.malloc((int) newCap, MemoryTag.NATIVE_DEFAULT); + if (sentDictBytesLen > 0) { + Unsafe.getUnsafe().copyMemory(sentDictBytesAddr, newAddr, sentDictBytesLen); + } + sentDictBytesAddr = newAddr; + sentDictBytesOwned = true; + } + sentDictBytesCapacity = (int) newCap; + } + + private void releaseSentDictBytes() { + if (sentDictBytesAddr != 0 && sentDictBytesOwned) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + } + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + sentDictBytesOwned = false; + sentDictCount = 0; + } + + private long readVarintAt(long p, long limit) { + long value = 0; + int shift = 0; + long cur = p; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(cur++); + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + // Defensive bound, matching PersistedSymbolDict.decodeVarint: a + // canonical entry-length / delta varint is <= 5 bytes. Every caller + // reads freshly-encoded, CRC- or openExisting-validated bytes, so + // this is unreachable, but it stops a corrupt continuation run from + // over-shifting into a garbage length. + break; + } + } + varintEnd = cur; + return value; + } + + /** + * Re-registers the whole symbol dictionary on a fresh connection, split into + * as many table-less frames as the server's advertised batch cap requires so + * no single frame exceeds it (a large dictionary would otherwise be rejected). + * Each chunk carries a contiguous id range {@code [start .. start+count)}, in + * order, so the server accumulates them exactly as it would the original + * per-frame deltas. Returns the number of frames sent (each consumed a wire + * sequence), so the caller can align {@code fsnAtZero}. Throws {@link + * CatchUpSendException} on a send error (retriable -- the caller reconnects); + * a single entry too large for the cap is non-retriable, so it latches a + * terminal before throwing. + */ + private int sendDictCatchUp() { + int cap = client.getServerMaxBatchSize(); + // The frame ceiling a catch-up chunk must not exceed: the server's + // advertised cap, or -- when the server advertises none (cap <= 0) -- + // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE + + // varints + symbolsLen) cannot overflow on a pathological multi-GB + // dictionary (unreachable at real cardinality; defensive). Used by the + // single-entry terminal below, which measures the real solo frame. + int frameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES; + // Symbol-bytes budget for PACKING several entries into one chunk, leaving + // room for the 12-byte header and the two delta-section varints. Kept + // deliberately conservative (reserving 16 for the varints): it only makes a + // multi-entry chunk split marginally earlier, never over the cap. It must + // NOT gate the single-entry terminal -- that reserve is larger than the + // minimal data-frame overhead, so an entry the producer already shipped + // under this cap could exceed the reserve yet still fit its own catch-up + // frame; the terminal tests the real solo frame against frameLimit instead. + int budget = cap > 0 + ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) + : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; + int framesSent = 0; + int chunkStartId = 0; + long chunkStartAddr = sentDictBytesAddr; + int chunkSymbols = 0; + long chunkBytes = 0; + long p = sentDictBytesAddr; + long limit = sentDictBytesAddr + sentDictBytesLen; + while (p < limit) { + long entryStart = p; + long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix + long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] + long entryBytes = entryEnd - entryStart; + // The exact table-less frame sendCatchUpChunk would build for THIS entry + // alone: header + deltaStart varint (the entry's own global id) + + // deltaCount varint (1) + the entry bytes. A cap gap exists only when even + // that solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // re-registered. Testing the real solo frame (not the conservative + // packing budget above) is what keeps a HOMOGENEOUS cluster + // livelock-free: an entry the producer already shipped in a data frame + // under this cap (header + delta varints + entry + schema + >=1 row) is + // strictly larger than its bare catch-up frame, so it always fits here. + long soloFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols) + + NativeBufferWriter.varintSize(1) + + entryBytes; + if (soloFrameLen > frameLimit) { + // Cap gap: this entry cannot be re-registered under the fresh + // server's advertised cap. A HOMOGENEOUS cluster never reaches here + // (an entry that fit its data frame under a cap always fits its bare + // catch-up frame under that same cap), so the only way in is a + // heterogeneous / rolling-cap failover to a smaller-cap node. + // + // A foreground sender retries indefinitely: store-and-forward must + // contain a transient heterogeneous-cluster window instead of surfacing + // it to the producer. Only an orphan drainer may apply the settle budget + // below and quarantine its slot after a persistent gap. Under budget the + // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with + // backoff and re-runs the catch-up, which resets the episode on a node + // that accepts it. An unrelated transport/upgrade/role state also resets + // the episode, so its downtime cannot satisfy the orphan dwell. On + // exhaustion latch via recordFatal, NOT fail() -- + // failing from inside the catch-up would re-enter connectLoop (see + // CatchUpSendException); the data must be resent after the cap is raised. + // Escalation needs BOTH the strike count AND a minimum wall-clock dwell + // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count + // alone measures how often we looked, not how long the gap has held -- + // and 16 attempts at the capped backoff take only ~2 minutes, less than + // an ordinary rolling restart of the larger-cap node. Escalating on the + // count alone would therefore hard-fail a live store-and-forward + // producer during a routine cluster operation. + // + // The STRIKE COUNT -- never the anchor's sign -- says whether an episode is + // already open. A System.nanoTime() instant is only meaningful as a + // difference: its origin is arbitrary and the spec permits negative values, + // so a sign test cannot tell an unset anchor from a real one. Wherever it + // misreads a real anchor as unset it re-anchors to now on EVERY strike, + // pinning episodeNanos at ~0, and the dwell can then never be satisfied -- + // the terminal never latches, however long the gap truly persists. + // recordHeadRejectionStrike keys its episode off poisonStrikes for the same + // reason. + if (catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { + throw new CatchUpSendException(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' + + "; retrying indefinitely -- a larger-cap node may return"), true); + } + + long nowNanos = System.nanoTime(); + if (catchUpCapGapAttempts == 0) { + catchUpCapGapFirstNanos = nowNanos; + } + catchUpCapGapAttempts++; + long episodeNanos = nowNanos - catchUpCapGapFirstNanos; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS + && episodeNanos >= catchUpCapGapMinEscalationWindowNanos; + LineSenderException err = new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + + ", episodeMillis=" + (episodeNanos / 1_000_000L) + + '/' + (catchUpCapGapMinEscalationWindowNanos / 1_000_000L) + ']' + + (exhausted + ? "; the data must be resent after the cap is raised" + : "; retrying -- a larger-cap node may return")); + if (exhausted) { + recordFatal(err); + } + throw new CatchUpSendException(err, true); + } + if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + chunkStartId += chunkSymbols; + chunkStartAddr = entryStart; + chunkSymbols = 0; + chunkBytes = 0; + } + chunkSymbols++; + chunkBytes += entryBytes; + p = entryEnd; + } + if (chunkSymbols > 0) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + } + // The whole dictionary re-registered without a cap gap: this node accepts + // every entry, so the cap-gap episode (if any) is over -- reset BOTH the strike + // count and the wall-clock anchor. Resetting only the count would leave a stale + // anchor from an old episode, so the very first strike of a LATER episode would + // already satisfy the dwell. + resetCatchUpCapGapEpisode(); + return framesSent; + } + + /** + * Sends one table-less catch-up frame carrying dictionary ids + * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link + * CatchUpSendException} on a send error instead of calling {@link #fail} + * (see that type for why the catch-up must not re-enter the reconnect loop); + * the caller turns it into a single, non-re-entrant reconnect. + */ + private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + // Compute the frame size in long and fail loud if it would overflow the int + // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each + // chunk's symbol bytes under the budget, so this is unreachable at real + // cardinality -- but the mirror-side ensureSentDictCapacity guards the same + // math, and a future caller must not be able to overflow this one silently. + long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + symbolsLen; + long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL; + if (frameLenL > MAX_SENT_DICT_BYTES) { + LineSenderException err = new LineSenderException( + "symbol dictionary catch-up frame exceeds the maximum size [" + + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + int payloadLen = (int) payloadLenL; + int frameLen = (int) frameLenL; + ensureCatchUpFrameCapacity(frameLen); + long frame = catchUpFrameAddr; + try { + Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); + Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); + Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); + Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); + Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO + // rows, so it must never trigger a server-side commit. Today it is + // always the first frame on a fresh (empty-transaction) connection, so + // committing nothing is a no-op -- but that invariant is load-bearing + // and unasserted. Deferring the (empty) commit removes the dependency: + // a future mid-stream catch-up cannot prematurely commit an in-flight + // deferred transaction. The dictionary delta still registers (as any + // deferred data frame's does); only the row commit is deferred, and the + // next real frame commits it. + Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT + | QwpConstants.FLAG_DEFER_COMMIT)); + Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount + Unsafe.getUnsafe().putInt(frame + 8, payloadLen); + long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); + q = NativeBufferWriter.writeVarint(q, deltaCount); + Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); + client.sendBinary(frame, frameLen); + } catch (Throwable t) { + // Do NOT fail() here -- see CatchUpSendException. Signal the failure + // up so exactly one non-re-entrant reconnect follows. A JVM Error is + // never a transient reconnect case; let it propagate as-is so the + // I/O loop latches it terminal rather than looping on it. + if (t instanceof Error) { + throw (Error) t; + } + throw new CatchUpSendException(t); + } + nextWireSeq++; // this catch-up chunk consumed a wire sequence + lastFrameOrPingNanos = System.nanoTime(); + totalFramesSent.incrementAndGet(); + } + + @TestOnly + public int catchUpFrameGrowthCount() { + return catchUpFrameGrowthCount; + } + + private void ensureCatchUpFrameCapacity(int required) { + if (catchUpFrameCapacity >= required) { + return; + } + long newCapacity = Math.max(required, Math.max(4096L, (long) catchUpFrameCapacity * 2L)); + if (newCapacity > MAX_SENT_DICT_BYTES) { + newCapacity = MAX_SENT_DICT_BYTES; + } + catchUpFrameAddr = Unsafe.realloc( + catchUpFrameAddr, + catchUpFrameCapacity, + (int) newCapacity, + MemoryTag.NATIVE_DEFAULT); + catchUpFrameCapacity = (int) newCapacity; + catchUpFrameGrowthCount++; + } + + private void freeCatchUpFrameBuffer() { + if (catchUpFrameAddr != 0L) { + Unsafe.free(catchUpFrameAddr, catchUpFrameCapacity, MemoryTag.NATIVE_DEFAULT); + catchUpFrameAddr = 0L; + catchUpFrameCapacity = 0; + } + } + private boolean tryReceiveAcks() { boolean any = false; try { @@ -1898,7 +2817,15 @@ private boolean trySendOne() { // Nothing sent on this connection yet: re-anchor in place past // the retired tail. The wireSeq<->FSN mapping is untouched // because no wire sequence has been consumed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // Re-anchor's catch-up send failed. fail() here is a fresh, + // non-re-entrant connectLoop entry from the I/O loop body -- + // the same recovery a normal trySendOne send failure takes. + fail(e.getCause()); + return false; + } return true; } // Frames were already sent on this connection: the linear @@ -1949,12 +2876,55 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // Torn-dictionary guard. sentDictCount is this loop's model of how many ids + // the CURRENT server has been told about: seeded from the persisted + // dictionary, re-registered by the catch-up, and extended by + // accumulateSentDict as each frame goes out. A frame whose delta starts ABOVE + // that coverage references ids the server was never given; replaying it would + // make the server null-pad the hole (QwpMessageCursor grows the dict with + // nulls to deltaStart+deltaCount) and land rows with SILENTLY NULL symbols. + // Fail terminally instead; the unreplayable data must be resent. + // + // The guard, the mirror and the catch-up are ONE mechanism and are all + // ungated (see the constructor). A frame at deltaStart == sentDictCount is + // contiguous and extends the coverage, so a slot whose frames start at id 0 + // replays cleanly with no persisted dictionary at all -- which is exactly why + // the mirror below must keep advancing even when the dictionary is missing. + // A gap here therefore means genuine loss: a host/power crash tore the + // unsynced .symbol-dict below the ids the surviving frames still reference + // (SF is process-crash but not host-crash durable). + int deltaStart = frameDeltaStart(frameAddr, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; + } try { - client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + client.sendBinary(frameAddr, payloadLen); } catch (Throwable t) { fail(t); return false; } + // A real ring frame (data or commit) has now gone out on this connection, + // as opposed to only the dictionary catch-up. onClose / handleServerRejection + // key their poison-strike vs pre-send decision off this, not off nextWireSeq + // (which the catch-up advances). + dataFrameSentThisConnection = true; + if (deltaStart >= 0) { + // Mirror the symbols this frame just registered on the server, so a later + // reconnect can rebuild the whole dictionary and the guard above keeps an + // accurate view of the server's coverage. Idempotent on replay: a frame + // whose delta we already hold advances nothing. + // + // Ungated (see the constructor): this is the ONLY thing that advances + // sentDictCount from the frames themselves, so gating it while leaving the + // guard ungated froze the coverage at 0 and terminal'd a slot that replays + // perfectly from id 0. + accumulateSentDict(frameAddr, payloadLen, deltaStart); + } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; @@ -1984,8 +2954,11 @@ void positionCursorForStart() { // starts past it. Zero wire cost, no recycle. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; + // Recovery / orphan-drain seed the dictionary mirror, so the initial + // connection may also need a catch-up (client is non-null in the + // sync-start and drainer paths; null in async-initial, where swapClient + // handles it on the first connect). + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } @@ -2023,19 +2996,71 @@ private boolean tryRetireOrphanTail() { return true; } + /** + * Determines whether an oversized symbol-dictionary catch-up entry is always + * retriable or may become terminal after the orphan settle budget. + */ + public enum CatchUpCapGapPolicy { + /** Keep reconnecting until a node with a compatible batch cap returns. */ + RETRY_FOREVER, + /** Quarantine an orphan slot after both the attempt and dwell limits expire. */ + TERMINAL_AFTER_SETTLE_BUDGET + } + + /** Identifies who owns data while the reconnect loop is unavailable. */ + public enum ReconnectPolicy { + /** A live producer keeps buffering and retries endpoint-policy failures. */ + FOREGROUND, + /** An orphan drainer returns terminal states to its quarantine owner. */ + ORPHAN + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close * the old client (if needed), build a new one with the same auth/TLS * config, connect, perform the WebSocket upgrade, and return it ready - * to send. Throw on a terminal failure (auth rejection, etc.) — the - * I/O loop will treat the throw as fatal. + * to send. The loop's {@link ReconnectPolicy} decides whether endpoint-policy + * failures are retried or returned as terminal. */ @FunctionalInterface public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; } + /** + * Signals that a symbol-dictionary catch-up frame could not be sent on the + * current connection. Thrown by {@link #sendDictCatchUp}/{@link + * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs + * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial + * connect, inside {@link #start()} / {@link #trySendOne} on the caller + * thread. Calling {@code fail()} from there would re-enter {@code + * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire + * mapping (a subsequent ACK then trims un-acked frames) and growing the + * stack until it overflows into a terminal, breaking the "retry a transient + * outage forever" invariant -- or run {@code connectLoop} on the caller + * thread and block {@code Sender} construction indefinitely. Each catch + * site instead turns it into ONE non-re-entrant reconnect: {@code + * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} + * from the I/O loop body (trySendOne path), or dropping the dead client so + * the I/O thread reconnects (start path). A JVM {@code Error} is never + * wrapped -- it must stay terminal. The {@code capGap} marker lets the reconnect + * loop preserve only consecutive incompatible-cap observations; ordinary catch-up + * send failures restart the orphan settle episode. + */ + private static final class CatchUpSendException extends RuntimeException { + private final boolean capGap; + + CatchUpSendException(Throwable cause) { + this(cause, false); + } + + CatchUpSendException(Throwable cause, boolean capGap) { + super(cause); + this.capGap = capGap; + } + } + /** * One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table * (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0) @@ -2173,7 +3198,7 @@ public void onClose(int code, String reason) { || code == WebSocketCloseCode.GOING_AWAY; LineSenderException cause = new LineSenderException( "WebSocket closed by server: code=" + code + " reason=" + reason); - if (!orderly && nextWireSeq > 0) { + if (!orderly && dataFrameSentThisConnection) { if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) { haltOnPoisonedFrame("ws-close[" + code + ' ' + WebSocketCloseCode.describe(code) + "]: " + reason, @@ -2280,17 +3305,21 @@ private void handleServerRejection(long wireSeq) { // value is only used to attribute an FSN to the error report -- // a rejection never advances the watermark. long highestSent = nextWireSeq - 1L; - if (highestSent < 0L) { - // Pre-send rejection: server emitted an error frame before - // we sent anything on this connection (typical after a - // fresh swapClient — auth failure, server-initiated halt, - // etc.). The server-named wireSeq does not correspond to - // any frame we sent, so clamping it to 0 and acknowledging - // fsnAtZero would silently advance ackedFsn past a real - // unsent batch (fsnAtZero == ackedFsn + 1 right after a - // swap). Skip the watermark advance entirely; still surface - // the error so the user's handler sees it and HALT errors - // remain producer-observable. + if (!dataFrameSentThisConnection) { + // Pre-send rejection: the server emitted an error frame before we + // sent any DATA frame on this connection (typical after a fresh + // swapClient -- auth failure, server-initiated halt, or a rejection + // of the dictionary catch-up itself). nextWireSeq may be > 0 here + // because the catch-up consumed wire sequences, so this keys off + // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a + // transient NACK of a catch-up frame would take the post-send + // poison-strike path and could escalate a transient outage to a + // terminal. The server-named wireSeq does not correspond to any + // data frame we sent, so clamping it to 0 and acknowledging + // fsnAtZero would silently advance ackedFsn past a real unsent + // batch. Skip the watermark advance entirely; still surface the + // error so the user's handler sees it and HALT errors remain + // producer-observable. handlePreSendRejection(wireSeq, status, category, policy); return; } @@ -2300,6 +3329,32 @@ private void handleServerRejection(long wireSeq) { wireSeq, highestSent); } long fsn = fsnAtZero + cappedSeq; + if (fsn <= engine.ackedFsn()) { + // The clamped wire seq maps at or below the replay head, so this + // NACK is for a dictionary catch-up frame -- which occupies the + // already-acked wire sequences below replayStart -- not a data frame + // this connection sent. (dataFrameSentThisConnection can be true here + // because trySendOne ships the head data frame before tryReceiveAcks + // reads the catch-up's NACK in the same loop iteration.) Attributing + // it a data FSN would key recordHeadRejectionStrike() off a + // below-baseline FSN -- negative when replayStart < catchUpFrames -- + // colliding with the poisonFsn == -1 "no suspect" sentinel and + // laundering a genuine poison run, and would report a bogus FSN. + // Treat it exactly like a pre-send rejection: surface + recycle, no + // poison strike, no watermark advance. Symmetric with the success + // path, where engine.acknowledge() no-ops at or below ackedFsn. A + // real replayed data frame is at fsn > ackedFsn, so it is never + // caught here. + // + // Catch-up frames therefore sit OUTSIDE the poison detector: a + // deterministically-NACKed catch-up recycles forever (paced, so no + // busy-loop). That is acceptable -- a catch-up only re-registers + // symbols the cluster already accepted, so a persistent NACK of one + // is a server bug, not a poison-frame the client can quarantine, and + // Invariant B's "retry a transient outage forever" applies. + handlePreSendRejection(wireSeq, status, category, policy); + return; + } // Best-effort table attribution: the parser populates // response.tableNames on error frames the same way it does on // STATUS_OK. If exactly one table was named, surface it; if diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..27523de4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -28,9 +28,11 @@ import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +46,7 @@ *

* On-disk layout — header and frame format: * 

- *   [u32 magic 'SF01'] [u8 ver=1] [u8 flags=0] [u16 reserved=0]
+ *   [u32 magic 'SF01'] [u8 ver]   [u8 flags=0] [u16 reserved=0]
  *   [u64 baseSeq]       [u64 createdMicros]                       24-byte header
  *   frame, frame, ...                                              each frame:
  *                                                                  [u32 crc32c]
@@ -63,7 +65,8 @@ public final class MmapSegment implements QuietCloseable {
     public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian
     public static final int FRAME_HEADER_SIZE = 8;   // u32 crc + u32 payloadLen
     public static final int HEADER_SIZE = 24;
-    public static final byte VERSION = 1;
+    public static final byte LEGACY_VERSION = 1;
+    public static final byte VERSION = 2;
     private static final int[] CRC32C_TABLE = buildCrc32cTable();
     private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class);
 
@@ -74,6 +77,7 @@ public final class MmapSegment implements QuietCloseable {
     // memory-backed segments — same cursor architecture, no disk involvement.
     // close() and msync() branch on this flag.
     private final boolean memoryBacked;
+    private final byte version;
     // appendCursor: written only by the producer thread, never read by anyone else
     // — it's the reservation cursor. Plain field is fine.
     private long appendCursor;
@@ -104,7 +108,7 @@ public final class MmapSegment implements QuietCloseable {
 
     private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
                         long baseSeq, long initialCursor, long frameCount,
-                        boolean memoryBacked, long tornTailBytes) {
+                        boolean memoryBacked, long tornTailBytes, byte version) {
         this.path = path;
         this.fd = fd;
         this.mmapAddress = mmapAddress;
@@ -115,6 +119,7 @@ private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
         this.frameCount = frameCount;
         this.memoryBacked = memoryBacked;
         this.tornTailBytes = tornTailBytes;
+        this.version = version;
     }
 
     /**
@@ -162,6 +167,50 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long
      * per-call {@code byte[]} + native-malloc the way the String overload does.
      */
     public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) {
+        return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, VERSION);
+    }
+
+    /**
+     * Creates a tiny v1 segment containing one deliberately non-QWP payload.
+     * Two such files with non-contiguous base sequences form the rollback
+     * barrier installed by {@link SegmentRing}: a v1-only reader opens both
+     * and fails its mandatory FSN-contiguity check before replay. If a process
+     * dies after creating only one guard, that reader still adopts a non-empty
+     * log and sends the invalid one-byte head frame first, so it cannot silently
+     * discard the v2 files and start a fresh slot.
+     */
+    static void createLegacyReaderGuard(String path, long baseSeq) {
+        long pathPtr = FilesFacade.INSTANCE.allocNativePath(path);
+        long payload = 0L;
+        try (MmapSegment segment = create(
+                FilesFacade.INSTANCE,
+                pathPtr,
+                path,
+                baseSeq,
+                HEADER_SIZE + FRAME_HEADER_SIZE + 1L,
+                LEGACY_VERSION
+        )) {
+            payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putByte(payload, (byte) 0);
+            if (segment.tryAppend(payload, 1) != HEADER_SIZE) {
+                throw new MmapSegmentException("could not append legacy-reader guard frame: " + path);
+            }
+        } finally {
+            if (payload != 0L) {
+                Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT);
+            }
+            FilesFacade.INSTANCE.freeNativePath(pathPtr);
+        }
+    }
+
+    private static MmapSegment create(
+            FilesFacade ff,
+            long pathPtr,
+            String displayPath,
+            long baseSeq,
+            long sizeBytes,
+            byte version
+    ) {
         if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) {
             throw new IllegalArgumentException(
                     "sizeBytes too small for header + one minimal frame: " + sizeBytes);
@@ -192,12 +241,13 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat
             }
             // Header goes straight into the mapping — no separate write syscall.
             Unsafe.getUnsafe().putInt(addr, FILE_MAGIC);
-            Unsafe.getUnsafe().putByte(addr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(addr + 4, version);
             Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
+            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, false, 0L, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
@@ -234,7 +284,8 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0);
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
+            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, true, 0L, VERSION);
         } catch (Throwable t) {
             Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT);
             throw t;
@@ -298,7 +349,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                         "bad magic in " + path + ": 0x" + Integer.toHexString(magic));
             }
             byte version = Unsafe.getUnsafe().getByte(addr + 4);
-            if (version != VERSION) {
+            if (version != LEGACY_VERSION && version != VERSION) {
                 throw new MmapSegmentException("unsupported version in " + path + ": " + version);
             }
             long baseSeq = Unsafe.getUnsafe().getLong(addr + 8);
@@ -313,8 +364,9 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                 throw new MmapSegmentException(
                         "bad baseSeq in " + path + ": " + baseSeq);
             }
-            long lastGood = scanFrames(addr, fileSize);
-            long count = countFrames(addr, lastGood);
+            FrameScan scan = scanFrames(addr, fileSize);
+            long lastGood = scan.lastGood;
+            long count = scan.frameCount;
             long tornTail = detectTornTail(addr, lastGood, fileSize);
             if (tornTail > 0) {
                 LOG.warn("SF segment {}: torn tail of {} bytes at offset {} "
@@ -324,7 +376,8 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                                 + "Investigate disk health or unexpected writer crash.",
                         path, tornTail, lastGood, fileSize, count);
             }
-            return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail);
+            return new MmapSegment(path, fd, addr, fileSize, baseSeq,
+                    lastGood, count, false, tornTail, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
@@ -438,6 +491,11 @@ public long sizeBytes() {
         return sizeBytes;
     }
 
+    /** On-disk format version read from or written to this segment. */
+    public byte version() {
+        return version;
+    }
+
     /**
      * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]}
      * starting at the current append cursor, then advances both cursors
@@ -522,6 +580,275 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in
         return best;
     }
 
+    /**
+     * Rebuilds, from the frames' own delta sections, the symbols a replay of
+     * {@code [minFsnInclusive .. maxFsnInclusive]} would register ABOVE {@code coverage},
+     * appending them to {@code out} in ascending id order. Returns the coverage the walk
+     * establishes (one past the highest id registered), or {@code -1} when a frame's delta
+     * starts ABOVE the coverage reached so far.
+     * 

+     * This is the recovery-time twin of {@code CursorWebSocketSendLoop.accumulateSentDict}:
+     * it visits the same frames, in the same FSN order, and extracts exactly the entries
+     * that method will add to the send loop's mirror as those frames go back on the wire.
+     * The producer's dictionary is seeded from it so the producer's delta baseline and the
+     * loop's mirror coverage stay in lockstep BY CONSTRUCTION -- which is what makes a slot
+     * whose {@code .symbol-dict} was torn (or lost entirely) recoverable, as long as the
+     * surviving frames define the ids themselves.
+     * 

+     * A {@code -1} return is a genuine GAP: the ids between the coverage and the frame's
+     * delta start were introduced by frames that were acked and trimmed away, so they lived
+     * ONLY in the lost dictionary and cannot be rebuilt from anything. That is the one case
+     * the data really must be resent, and it is exactly the condition the send loop's
+     * torn-dictionary guard ({@code deltaStart > sentDictCount}) would trip on at replay --
+     * so the two agree, rather than the producer failing on slots the drainer drains fine.
+     * 

+     * Same frame layout and FSN bound as {@link #maxSymbolDeltaEnd}.
+     */
+    public long collectReplaySymbolsAbove(
+            int headerMagic,
+            int flagsOffset,
+            int flagDeltaMask,
+            int qwpHeaderSize,
+            long minFsnInclusive,
+            long maxFsnInclusive,
+            long coverage,
+            ObjList out
+    ) {
+        long off = HEADER_SIZE;
+        long frames = frameCount;
+        for (long i = 0; i < frames; i++) {
+            long fsn = baseSeq + i;
+            int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
+            long payload = mmapAddress + off + FRAME_HEADER_SIZE;
+            if (fsn >= minFsnInclusive
+                    && fsn <= maxFsnInclusive
+                    && payloadLen >= qwpHeaderSize
+                    && payloadLen > flagsOffset
+                    && Unsafe.getUnsafe().getInt(payload) == headerMagic
+                    && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) {
+                long p = payload + qwpHeaderSize;
+                long limit = payload + payloadLen;
+                long deltaStart = 0L;
+                int shift = 0;
+                while (p < limit) {
+                    byte b = Unsafe.getUnsafe().getByte(p++);
+                    deltaStart |= (long) (b & 0x7F) << shift;
+                    if ((b & 0x80) == 0) {
+                        break;
+                    }
+                    shift += 7;
+                    if (shift > 35) {
+                        break; // corrupt run; recovered frames are CRC-valid, so defensive only
+                    }
+                }
+                long deltaCount = 0L;
+                shift = 0;
+                while (p < limit) {
+                    byte b = Unsafe.getUnsafe().getByte(p++);
+                    deltaCount |= (long) (b & 0x7F) << shift;
+                    if ((b & 0x80) == 0) {
+                        break;
+                    }
+                    shift += 7;
+                    if (shift > 35) {
+                        break;
+                    }
+                }
+                if (deltaStart > coverage) {
+                    return -1L; // gap: ids [coverage, deltaStart) are unrecoverable
+                }
+                // Walk this frame's [len varint][utf8] entries. Ids below the coverage we
+                // already hold are skipped (a replayed frame legitimately overlaps the tip);
+                // ids at or above it extend the dictionary.
+                long id = deltaStart;
+                for (long k = 0; k < deltaCount; k++, id++) {
+                    long len = 0L;
+                    shift = 0;
+                    while (p < limit) {
+                        byte b = Unsafe.getUnsafe().getByte(p++);
+                        len |= (long) (b & 0x7F) << shift;
+                        if ((b & 0x80) == 0) {
+                            break;
+                        }
+                        shift += 7;
+                        if (shift > 35) {
+                            break;
+                        }
+                    }
+                    if (p + len > limit) {
+                        // Malformed entry region. Do NOT guess: report it as unrecoverable
+                        // so the caller fails clean rather than seeding a shifted dictionary.
+                        return -1L;
+                    }
+                    if (id >= coverage) {
+                        out.add(Utf8s.stringFromUtf8Bytes(p, p + len));
+                    }
+                    p += len;
+                }
+                long deltaEnd = deltaStart + deltaCount;
+                if (deltaEnd > coverage) {
+                    coverage = deltaEnd;
+                }
+            }
+            off += FRAME_HEADER_SIZE + payloadLen;
+        }
+        return coverage;
+    }
+
+    /**
+     * Feeds every recovered frame in this segment to the engine's single-pass
+     * recovery fold. Segments are visited oldest-first by {@link SegmentRing}.
+     */
+    void scanRecovery(RecoveredFrameAnalysis analysis) {
+        long off = HEADER_SIZE;
+        long frames = frameCount;
+        for (long i = 0; i < frames; i++) {
+            int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
+            analysis.accept(baseSeq + i, mmapAddress + off + FRAME_HEADER_SIZE, payloadLen);
+            off += FRAME_HEADER_SIZE + payloadLen;
+        }
+    }
+
+    /**
+     * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any
+     * delta-flagged frame in this segment carries, or {@code 0} only when NO
+     * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit
+     * or a symbol-reusing frame -- the encoder always sets the delta flag) still
+     * contributes its {@code deltaStart}, i.e. the producer's baseline at encode
+     * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that
+     * baseline so a dictionary torn below it is detected -- a conservative
+     * over-strand (it may over-reject a frame whose rows reference only surviving
+     * ids), never an under-strand that could silently shift the dense id map.
+     * Read-only walk over the recovered frames, used
+     * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash,
+     * out-of-order page loss) below the ids the surviving frames still reference: if
+     * the max here reaches at or beyond the recovered dictionary size, a resuming
+     * producer -- seeded from the shorter dictionary -- would re-use ids the frames
+     * already define. Only frames at or below {@code maxFsnInclusive} count: frames
+     * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires
+     * without ever transmitting, so their ids must not inflate the result (a resuming
+     * producer never reuses them on the wire). The frame layout mirrors
+     * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header
+     * ({@code qwpHeaderSize} bytes) is followed by the delta section
+     * {@code [deltaStart varint][deltaCount varint]...}.
+     *
+     * @param headerMagic     the QWP message magic identifying a well-formed frame
+     * @param flagsOffset     byte offset of the flags field within the QWP header
+     * @param flagDeltaMask   the FLAG_DELTA_SYMBOL_DICT bit
+     * @param qwpHeaderSize   the QWP message header size (delta section starts past it)
+     * @param maxFsnInclusive highest frame FSN to consider; frames above it are the
+     *                        retired orphan-deferred tail -- pass
+     *                        {@code recoveredCommitBoundaryFsn}
+     */
+    public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) {
+        long maxEnd = 0L;
+        long off = HEADER_SIZE;
+        long frames = frameCount;
+        for (long i = 0; i < frames; i++) {
+            long fsn = baseSeq + i;
+            int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
+            long payload = mmapAddress + off + FRAME_HEADER_SIZE;
+            // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never
+            // sent, so its ids must not count. off still advances below.
+            if (fsn <= maxFsnInclusive
+                    && payloadLen >= qwpHeaderSize
+                    && payloadLen > flagsOffset
+                    && Unsafe.getUnsafe().getInt(payload) == headerMagic
+                    && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) {
+                long p = payload + qwpHeaderSize;
+                long limit = payload + payloadLen;
+                long deltaStart = 0L;
+                int shift = 0;
+                while (p < limit) {
+                    byte b = Unsafe.getUnsafe().getByte(p++);
+                    deltaStart |= (long) (b & 0x7F) << shift;
+                    if ((b & 0x80) == 0) {
+                        break;
+                    }
+                    shift += 7;
+                    if (shift > 35) {
+                        break; // corrupt run; recovered frames are CRC-valid, so defensive only
+                    }
+                }
+                long deltaCount = 0L;
+                shift = 0;
+                while (p < limit) {
+                    byte b = Unsafe.getUnsafe().getByte(p++);
+                    deltaCount |= (long) (b & 0x7F) << shift;
+                    if ((b & 0x80) == 0) {
+                        break;
+                    }
+                    shift += 7;
+                    if (shift > 35) {
+                        break;
+                    }
+                }
+                long end = deltaStart + deltaCount;
+                if (end > maxEnd) {
+                    maxEnd = end;
+                }
+            }
+            off += FRAME_HEADER_SIZE + payloadLen;
+        }
+        return maxEnd;
+    }
+
+    /**
+     * Highest {@code deltaStart} any delta-flagged frame at or below
+     * {@code maxFsnInclusive} carries, or {@code 0} when NO delta-flagged frame is
+     * present. A result of {@code 0} means every surviving symbol-bearing frame is
+     * SELF-SUFFICIENT -- it re-registers its dictionary from id 0 (a full-dict
+     * frame), so the slot replays with no persisted dictionary at all; a result
+     * {@code > 0} means at least one frame is a non-self-sufficient delta whose ids
+     * depend on a prior frame's (or the persisted dictionary's) registrations.
+     * {@link CursorSendEngine} pairs this with {@link #maxSymbolDeltaEnd} at
+     * recovery: when the persisted {@code .symbol-dict} is a subset of the ids the
+     * frames reference BUT this returns {@code 0}, the slot was written in full-dict
+     * fallback (the dictionary never opened when writing) and recovers in full-dict
+     * mode instead of failing clean; a torn DELTA dictionary has a {@code
+     * deltaStart > 0} frame and still fails clean. Same read-only frame walk and
+     * layout as {@link #maxSymbolDeltaEnd}; only frames at or below {@code
+     * maxFsnInclusive} count (the retired orphan-deferred tail is excluded).
+     */
+    public long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) {
+        long maxStart = 0L;
+        long off = HEADER_SIZE;
+        long frames = frameCount;
+        for (long i = 0; i < frames; i++) {
+            long fsn = baseSeq + i;
+            int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
+            long payload = mmapAddress + off + FRAME_HEADER_SIZE;
+            // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never
+            // sent, so its baseline must not count. off still advances below.
+            if (fsn <= maxFsnInclusive
+                    && payloadLen >= qwpHeaderSize
+                    && payloadLen > flagsOffset
+                    && Unsafe.getUnsafe().getInt(payload) == headerMagic
+                    && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) {
+                long p = payload + qwpHeaderSize;
+                long limit = payload + payloadLen;
+                long deltaStart = 0L;
+                int shift = 0;
+                while (p < limit) {
+                    byte b = Unsafe.getUnsafe().getByte(p++);
+                    deltaStart |= (long) (b & 0x7F) << shift;
+                    if ((b & 0x80) == 0) {
+                        break;
+                    }
+                    shift += 7;
+                    if (shift > 35) {
+                        break; // corrupt run; recovered frames are CRC-valid, so defensive only
+                    }
+                }
+                if (deltaStart > maxStart) {
+                    maxStart = deltaStart;
+                }
+            }
+            off += FRAME_HEADER_SIZE + payloadLen;
+        }
+        return maxStart;
+    }
+
     /**
      * Number of frames written since {@link #create} (or recovered by
      * {@link #openExisting}). Used by {@code SegmentRing} to compute
@@ -581,14 +908,15 @@ private static boolean isMmapAccessFault(Throwable t) {
     }
 
     /**
-     * Forward scan that returns the offset just past the last frame whose
-     * CRC verifies. A torn-tail frame (declared length runs past EOF, or
-     * CRC mismatch) leaves both cursors at the start of that frame; the
-     * next {@link #tryAppend} will overwrite it. The scan only reads from
-     * the mapping — no syscalls.
+     * Forward scan that returns the offset just past the last frame whose CRC
+     * verifies and the number of verified frames. A torn-tail frame (declared
+     * length runs past EOF, or CRC mismatch) leaves both cursors at the start of
+     * that frame; the next {@link #tryAppend} will overwrite it. The scan only
+     * reads from the mapping — no syscalls.
      */
-    private static long scanFrames(long addr, long fileSize) {
+    private static FrameScan scanFrames(long addr, long fileSize) {
         long pos = HEADER_SIZE;
+        long frameCount = 0L;
         try {
             while (pos + FRAME_HEADER_SIZE <= fileSize) {
                 int crcRead = Unsafe.getUnsafe().getInt(addr + pos);
@@ -596,7 +924,7 @@ private static long scanFrames(long addr, long fileSize) {
                 // Defensive: a corrupt length field could be enormous or negative,
                 // both of which would otherwise overrun the mapping.
                 if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 // CRC over the contiguous (payloadLen, payload) pair, folded
                 // via Unsafe reads rather than the native Crc32c.update.
@@ -619,9 +947,10 @@ private static long scanFrames(long addr, long fileSize) {
                 // table CRC here is immaterial.
                 int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen);
                 if (crcCalc != crcRead) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 pos += FRAME_HEADER_SIZE + payloadLen;
+                frameCount++;
             }
         } catch (InternalError e) {
             // The read at `pos` hit a mapped page that is not backed by real
@@ -647,7 +976,7 @@ private static long scanFrames(long addr, long fileSize) {
                             + "if this recurs.",
                     pos, fileSize);
         }
-        return pos;
+        return new FrameScan(pos, frameCount);
     }
 
     /**
@@ -723,19 +1052,13 @@ private static long detectTornTail(long addr, long lastGood, long fileSize) {
         return 0L;
     }
 
-    /**
-     * Counts frames in {@code [HEADER_SIZE, lastGood)}. Walks the framing in
-     * lockstep with {@link #scanFrames} (which already validated CRCs); so
-     * this is just length-driven traversal, no CRC re-check.
-     */
-    private static long countFrames(long addr, long lastGood) {
-        long pos = HEADER_SIZE;
-        long count = 0;
-        while (pos < lastGood) {
-            int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4);
-            pos += FRAME_HEADER_SIZE + payloadLen;
-            count++;
+    private static final class FrameScan {
+        private final long frameCount;
+        private final long lastGood;
+
+        private FrameScan(long lastGood, long frameCount) {
+            this.lastGood = lastGood;
+            this.frameCount = frameCount;
         }
-        return count;
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
new file mode 100644
index 00000000..914e4ab5
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
@@ -0,0 +1,1070 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.std.Crc32c;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+import org.jetbrains.annotations.TestOnly;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Append-only, per-slot persistence of the global symbol dictionary that a
+ * store-and-forward sender ships to the server with delta encoding. Lives at
+ * {@code /.symbol-dict} alongside the segment files, the slot lock and
+ * {@code .ack-watermark}.
+ * 

+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the
+ * symbols it introduces, so recovering (process restart) or draining (orphan
+ * adoption) a slot requires re-registering the whole dictionary on the fresh
+ * server before those frames replay. This file is that dictionary. Unlike
+ * {@link AckWatermark} -- a discardable optimization protected by a
+ * {@code max()} clamp -- this file is load-bearing: a surviving frame
+ * that references an id missing from it is unrecoverable. It is therefore held
+ * to a stronger durability contract, and {@link #open} never destroys it (see
+ * "Never recreate over an existing file" below).
+ * 

+ * Layout (little-endian):
+ * 
+ *   offset 0: u32 magic = 'SYD1'
+ *   offset 4: u8  version = 3
+ *   offset 5: 3 bytes reserved (zero)
+ *   offset 8: chunks, each
+ *             [entryCount: varint][entryBytes: varint][entries][crc32c: u32]
+ *             where entries = [len: varint][utf8] repeated entryCount times,
+ *             occupying exactly entryBytes bytes, and the CRC-32C covers the
+ *             two header varints AND the entry region.
+ * 

+ * A chunk is one append -- i.e. exactly the set of symbols one frame
+ * introduces, since the producer persists a frame's new symbols in a single call
+ * before publishing it. Symbol id {@code i} is the {@code i}-th entry across all
+ * chunks (ids are dense and assigned sequentially from 0), so no id is stored.
+ * 

+ * Why the checksum is per chunk, not per entry. The only consumer of the
+ * recovered prefix is the send loop's replay guard, which compares a surviving
+ * frame's {@code deltaStart} against the recovered dictionary size -- and every
+ * {@code deltaStart} is a chunk boundary, because chunks and frame deltas are
+ * written one-for-one. A tear inside a chunk therefore invalidates exactly the
+ * frames a per-entry checksum would have invalidated anyway: per-entry
+ * granularity buys no extra recoverable prefix. It costs a great deal, though.
+ * {@link Crc32c#update} is a native call, so checksumming per entry put one JNI
+ * transition -- plus one sub-cache-line copy and one redundant varint decode --
+ * on the producer thread for every new symbol. On the high-cardinality batch this
+ * feature exists to serve (one new symbol per row), that is a thousand native
+ * calls per flush where the chunk needs one.
+ * 

+ * Durability / write-ahead ordering: the producer appends the symbols a
+ * frame introduces BEFORE that frame is published to the ring, but does NOT
+ * fsync -- matching the rest of store-and-forward, which is page-cache (not
+ * disk) durable. This ordering is sufficient for a process/JVM crash: the
+ * page cache survives, so both the dictionary and the frames survive and the
+ * dictionary is a superset of every recoverable frame's references. It is NOT
+ * sufficient for a host/power crash, where unflushed pages can be lost out
+ * of order and the dictionary may end up torn relative to the frames it serves --
+ * exactly as the segment frames themselves may be lost on a host crash. Two
+ * layers keep a host-crash tear from silently corrupting data:
+ * 
    
+ *   
  • The per-chunk CRC-32C: {@link #open} verifies every chunk and stops at
+ *       the first one whose checksum fails, so an interior page lost out of
+ *       order (reading back as zeroes) or a stale chunk left past the end by a
+ *       failed truncate is DETECTED and the trusted region ends before it --
+ *       recovery never mis-parses a corrupt chunk as real symbols nor shifts the
+ *       dense id->symbol map.
    • 
+ *   
  • The send loop's replay guard: once recovery trusts only the intact
+ *       prefix, a surviving frame whose delta start id exceeds that prefix
+ *       fails loudly (the unreplayable data must be resent) rather than sending
+ *       a gapped frame.
    • 
+ * 

+ * Together these turn every detectable host-crash tear into a fail-clean
+ * "resend required" instead of a silent symbol misattribution -- the same
+ * CRC-32C protection the segment frames carry. A tear that happened to leave a
+ * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32
+ * collision per corrupted chunk, no weaker than the frames' own checksum.
+ * 

+ * A torn trailing chunk from a crash mid-append is self-healing: {@link #open}
+ * stops parsing at the first incomplete chunk and the next append overwrites it.
+ * 

+ * Never recreate over an existing file. {@link #open} -- the RECOVERY
+ * entry point -- returns {@code null} when an existing file cannot be read or
+ * parsed, and NEVER falls back to recreating it empty. Recreating would mean
+ * {@code O_TRUNC} over the only copy of load-bearing state, so a single transient
+ * read error (an EIO on a flaky disk, a short read) would permanently destroy the
+ * dictionary the surviving delta frames reference -- turning a recoverable outage
+ * into unrecoverable data. A {@code null} instead degrades the sender to full
+ * self-sufficient frames and leaves every byte on disk, so a later attempt, once
+ * the transient clears, can still recover the slot in full. Only
+ * {@link #openClean} -- the FRESH-slot path, where discarding is the whole point
+ * -- truncates.
+ * 

+ * Lifecycle: single-writer (the producer / user thread) for appends. Read
+ * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The
+ * owner (the engine) closes it, and {@code close()} is callable from any thread
+ * (a shutdown hook, test cleanup). {@code close()} and the append methods are
+ * therefore {@code synchronized}: without that, a close racing an in-flight append
+ * could unmap the production append region (or free the fault-test scratch buffer),
+ * close the fd, and let an in-flight write corrupt memory or land on a descriptor
+ * the OS has reused for another file. Not thread-safe for concurrent writers.
+ */
+public final class PersistedSymbolDict implements QuietCloseable {
+
+    /**
+     * Filename within the slot directory. Dot-prefixed so directory
+     * enumerators that filter by the {@code .sfa} suffix (segment recovery,
+     * OrphanScanner, trim) skip it automatically.
+     */
+    public static final String FILE_NAME = ".symbol-dict";
+    static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk
+    static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian
+    static final int HEADER_SIZE = 8;
+    static final int INITIAL_APPEND_MAP_CAPACITY = 64 * 1024;
+    /**
+     * Upper bound on a chunk's two header varints ({@code entryCount} and
+     * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The
+     * encoders reserve this much in front of the entry region so the header can
+     * be back-filled once the region's exact size is known, keeping header,
+     * entries and CRC one contiguous run.
+     */
+    static final int MAX_CHUNK_HEADER_SIZE = 10;
+    /**
+     * Ceiling for the append scratch buffer, mirroring
+     * {@code CursorWebSocketSendLoop.MAX_SENT_DICT_BYTES}: the capacity math is
+     * int-typed, so a larger buffer cannot be addressed. Exceeding it throws --
+     * {@link #ensureScratch} never silently under-allocates.
+     */
+    static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8;
+    static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk
+    private static final int[] CRC32C_TABLE = buildCrc32cTable();
+    private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class);
+    private final int fd;
+    // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files);
+    // tests inject a fault facade to exercise recovery I/O failures (a truncate
+    // that cannot drop a torn tail, a short write) without a real broken disk.
+    private final FilesFacade ff;
+    // Production writes directly into segmented append mappings. Custom facades retain the
+    // positioned-write path so fault tests can inject short writes through ff.write.
+    private final boolean mappedAppend;
+    // True only when recovery parsed the file through a temporary read-only
+    // mmap instead of allocating a second native buffer as large as the file.
+    // Test-visible so the peak-memory regression has an observable contract.
+    private final boolean mappedRecoveryInput;
+    private long appendMapAddr;
+    private long appendMapCapacity;
+    private long appendMapOffset;
+    private int appendMapGrowthCount;
+    private long appendOffset;
+    private long appendWriteCount;
+    private boolean closed;
+    // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and
+    // CRCs stripped -- populated only when open() recovered existing chunks
+    // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not
+    // consumed) to seed the producer's id map and to seed the send loop's catch-up
+    // mirror. Foreground construction transfers ownership to its sole loop after
+    // producer seeding; orphan-drainer loops borrow it because one engine may create
+    // several wire sessions. If ownership was not transferred, close() frees it.
+    private long loadedEntriesAddr;
+    private int loadedEntriesLen;
+    private long scratchAddr;
+    private int scratchCap;
+    private int size;
+
+    private PersistedSymbolDict(
+            FilesFacade ff,
+            int fd,
+            long appendOffset,
+            int size,
+            long loadedEntriesAddr,
+            int loadedEntriesLen,
+            boolean mappedRecoveryInput
+    ) {
+        this.ff = ff;
+        this.fd = fd;
+        this.mappedAppend = ff == FilesFacade.INSTANCE;
+        this.mappedRecoveryInput = mappedRecoveryInput;
+        this.appendOffset = appendOffset;
+        this.size = size;
+        this.loadedEntriesAddr = loadedEntriesAddr;
+        this.loadedEntriesLen = loadedEntriesLen;
+    }
+
+    /**
+     * Opens the dictionary file in {@code slotDir} for RECOVERY, creating it only
+     * when it does not already exist. An existing file is parsed and its complete,
+     * CRC-valid chunks are loaded into memory (see {@link #loadedEntriesAddr()}).
+     * 

+     * Returns {@code null} on any I/O or parse failure -- including an existing file
+     * that cannot be read, carries an unknown version, or fails its checksums. The
+     * caller then falls back to full-dictionary (self-sufficient) frames for this
+     * slot, so a broken side-file degrades gracefully rather than aborting the
+     * sender. Crucially, a {@code null} return NEVER destroys the file: see the
+     * class-level "Never recreate over an existing file" note.
+     */
+    public static PersistedSymbolDict open(String slotDir) {
+        return open(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #open(String)}. Production passes
+     * {@link FilesFacade#INSTANCE}; tests inject a fault facade to drive recovery
+     * I/O failures (e.g. a truncate that cannot drop a torn tail).
+     */
+    public static PersistedSymbolDict open(FilesFacade ff, String slotDir) {
+        String filePath = slotDir + "/" + FILE_NAME;
+        boolean exists = ff.exists(filePath);
+        long existing = exists ? ff.length(filePath) : -1L;
+        if (exists && existing < 0) {
+            // The file is present but its length could not be stat'd (a transient EIO
+            // on a flaky disk). Do NOT fall through to openFresh below, which O_TRUNCs:
+            // truncating the only copy of load-bearing state on a TRANSIENT fault is the
+            // exact destruction the class-level "Never recreate over an existing file"
+            // note forbids -- and unlike the openExisting read path, this routing check
+            // otherwise has no guard. A genuine sub-header stub reports a length in
+            // [0, HEADER_SIZE); only a stat error reports < 0, so the two are
+            // distinguishable. Degrade to full self-sufficient frames and leave every
+            // byte on disk for a later attempt, once the transient clears.
+            LOG.warn("symbol dict {} exists but its length could not be read; "
+                    + "falling back to full-dictionary frames (file left intact)", filePath);
+            return null;
+        }
+        if (existing >= HEADER_SIZE) {
+            // Chunk lengths and the retained contiguous entry region are int-sized,
+            // so a dictionary at or past Integer.MAX_VALUE cannot be represented
+            // safely even though production recovery maps rather than reads it.
+            if (existing >= Integer.MAX_VALUE) {
+                LOG.warn("symbol dict {} too large ({} bytes) to reopen; "
+                        + "falling back to full-dictionary frames (file left intact)", filePath, existing);
+                return null;
+            }
+            // NEVER recreate over an existing file on the recovery path: openFresh
+            // truncates, and these bytes are the only copy of state the surviving
+            // delta frames reference. A null degrades this slot to full
+            // self-sufficient frames and preserves the file for a later attempt.
+            return openExisting(ff, filePath, existing);
+        }
+        // Absent, or a sub-header stub left by a crash inside openFresh: no
+        // load-bearing content to lose, so create it.
+        return openFresh(ff, filePath);
+    }
+
+    /**
+     * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding
+     * any surviving content. This is the fresh-start counterpart to {@link #open}:
+     * a slot with no recovered segments must start with an empty dictionary, so a
+     * dictionary left by a prior lifecycle -- a fully-drained slot whose
+     * best-effort delete failed, or a crash in the close window -- must NOT be
+     * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for
+     * recovery/orphan-drain replay and never destroys it, this truncates: the
+     * fresh-start producer is not seeded from the dictionary, so trusting a survivor
+     * would leave the producer's ids diverged from the dictionary the send loop
+     * replays and silently misattribute symbols on the next reconnect. Truncating
+     * (rather than relying on an unlink succeeding first) closes the gap even when
+     * the delete is refused -- e.g. a Windows share lock. Returns {@code null} on
+     * I/O failure, so the caller falls back to full self-sufficient frames exactly
+     * as {@link #open} does.
+     */
+    public static PersistedSymbolDict openClean(String slotDir) {
+        return openClean(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #openClean(String)}.
+     */
+    public static PersistedSymbolDict openClean(FilesFacade ff, String slotDir) {
+        return openFresh(ff, slotDir + "/" + FILE_NAME);
+    }
+
+    /**
+     * Best-effort removal of a stale dictionary file. Used at fully-drained close
+     * (the slot is empty, nothing references the dictionary any more), mirroring
+     * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT
+     * use this -- it opens a clean dictionary via {@link #openClean} instead, so a
+     * failed delete cannot leave a stale dictionary a new session would trust.
+     */
+    public static void removeOrphan(String slotDir) {
+        removeOrphan(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #removeOrphan(String)}.
+     */
+    public static void removeOrphan(FilesFacade ff, String slotDir) {
+        ff.remove(slotDir + "/" + FILE_NAME);
+    }
+
+    /**
+     * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the
+     * symbol-dict delta section the frame encoder just wrote -- as ONE chunk.
+     * 

+     * The consistency walk below decodes each entry's length varint, but the bytes
+     * themselves are copied in a SINGLE {@code copyMemory} and checksummed by a
+     * SINGLE {@link Crc32c#update} covering the whole chunk. A per-entry checksum
+     * would put one JNI transition, one sub-cache-line copy and one redundant varint
+     * decode on the producer thread per new symbol; the chunk needs one of each.
+     * 

+     * Advances {@code size} by {@code count}. Same durability/idempotency contract
+     * as {@link #appendSymbols}: no fsync, and a short write throws WITHOUT
+     * advancing {@code size}/{@code appendOffset}, so a retry keyed off
+     * {@link #size()} re-persists the same range at the same offset. No-op when the
+     * range is empty or the dictionary is closed.
+     */
+    public synchronized void appendRawEntries(long addr, int len, int count) {
+        if (closed || count <= 0 || len <= 0) {
+            return;
+        }
+        // Validate the (addr, len, count) triple BEFORE writing anything: an
+        // inconsistent triple would record a chunk whose stored entryCount disagreed
+        // with the entries it holds, shifting the dense id->symbol map on recovery.
+        // The sole caller derives count and len from one beginMessage, so this cannot
+        // fire today -- but the file this guards is the one the "resend required"
+        // contract rests on, so keep the structural guard. Gated behind assert: it
+        // re-walks every entry's length prefix on the common flush path, and the client
+        // library runs without -ea in production (embedded in user apps), so this holds
+        // the guard in the client's own -ea test suite without the per-flush cost in
+        // production.
+        assert validateRawEntries(addr, len, count);
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len);
+        if (mappedAppend) {
+            long recLen = (long) hdrLen + len + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            NativeBufferWriter.writeVarint(p, len);
+            Unsafe.getUnsafe().copyMemory(addr, recStart + hdrLen, len);
+            commitMappedChunk(recStart, hdrLen, len, count);
+            return;
+        }
+        ensureScratch((long) hdrLen + len + CRC_SIZE);
+        long p = NativeBufferWriter.writeVarint(scratchAddr, count);
+        NativeBufferWriter.writeVarint(p, len);
+        Unsafe.getUnsafe().copyMemory(addr, scratchAddr + hdrLen, len);
+        flushChunk(scratchAddr, hdrLen, len, count);
+    }
+
+    /**
+     * Appends one symbol as a single-entry chunk, extending the on-disk dictionary.
+     * The caller appends a frame's new symbols BEFORE publishing that frame, so the
+     * write ordering (dictionary entry before referencing frame) holds; no fsync is
+     * performed (see the class-level durability note). Assigns the next dense id
+     * implicitly (the entry's position).
+     * 

+     * Test-only: production persists a frame's whole new-symbol range in one chunk
+     * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to
+     * build a dictionary one entry at a time.
+     */
+    @TestOnly
+    public synchronized void appendSymbol(CharSequence symbol) {
+        if (closed) {
+            return;
+        }
+        int utf8Len = Utf8s.utf8Bytes(symbol);
+        int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+        if (mappedAppend) {
+            int hdrLen = NativeBufferWriter.varintSize(1) + NativeBufferWriter.varintSize(wireLen);
+            long recLen = (long) hdrLen + wireLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, 1);
+            p = NativeBufferWriter.writeVarint(p, wireLen);
+            p = NativeBufferWriter.writeVarint(p, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            commitMappedChunk(recStart, hdrLen, wireLen, 1);
+            return;
+        }
+        ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE);
+        long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE;
+        long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+        if (utf8Len > 0) {
+            Utf8s.strCpyUtf8(symbol, p, utf8Len);
+        }
+        writeChunkFromScratch(wireLen, 1);
+    }
+
+    /**
+     * Appends the dense id range {@code [from .. to]} as one chunk with one
+     * checksum. Production encodes directly into a segmented append mapping, avoiding
+     * both the staging copy and a positioned-write syscall on every flush. The
+     * scratch/positioned-write fallback exists only behind an injected filesystem
+     * facade so short-write recovery tests retain their fault seam. Callers pass the
+     * dictionary and the range so the ids resolve to their symbol strings.
+     * 

+     * Same durability and idempotency contract as {@link #appendSymbol}: no fsync,
+     * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset},
+     * so a retry keyed off {@link #size()} re-encodes the same range and overwrites
+     * at the same offset. No-op when the range is empty or the dictionary is closed.
+     */
+    public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) {
+        if (closed || to < from) {
+            return;
+        }
+        int count = to - from + 1;
+        if (mappedAppend) {
+            long entriesLenLong = 0L;
+            for (int id = from; id <= to; id++) {
+                int utf8Len = Utf8s.utf8Bytes(dict.getSymbol(id));
+                entriesLenLong += NativeBufferWriter.varintSize(utf8Len) + (long) utf8Len;
+                if (entriesLenLong > MAX_SCRATCH_BYTES) {
+                    throw new IllegalStateException("symbol dict chunk exceeds the maximum size to "
+                            + FILE_NAME + " [required=" + entriesLenLong + ", max="
+                            + MAX_SCRATCH_BYTES + ']');
+                }
+            }
+            int entriesLen = (int) entriesLenLong;
+            int hdrLen = NativeBufferWriter.varintSize(count)
+                    + NativeBufferWriter.varintSize(entriesLen);
+            long recLen = (long) hdrLen + entriesLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            p = NativeBufferWriter.writeVarint(p, entriesLen);
+            for (int id = from; id <= to; id++) {
+                CharSequence symbol = dict.getSymbol(id);
+                int utf8Len = Utf8s.utf8Bytes(symbol);
+                p = NativeBufferWriter.writeVarint(p, utf8Len);
+                if (utf8Len > 0) {
+                    Utf8s.strCpyUtf8(symbol, p, utf8Len);
+                    p += utf8Len;
+                }
+            }
+            commitMappedChunk(recStart, hdrLen, entriesLen, count);
+            return;
+        }
+        int entriesLen = 0;
+        for (int id = from; id <= to; id++) {
+            CharSequence symbol = dict.getSymbol(id);
+            int utf8Len = Utf8s.utf8Bytes(symbol);
+            int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+            ensureScratch((long) MAX_CHUNK_HEADER_SIZE + entriesLen + wireLen + CRC_SIZE);
+            long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE + entriesLen;
+            long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            entriesLen += wireLen;
+        }
+        writeChunkFromScratch(entriesLen, count);
+    }
+
+    @Override
+    public synchronized void close() {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        if (loadedEntriesAddr != 0L) {
+            Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT);
+            // Null after freeing (like scratchAddr below) so a future accessor that
+            // reads loadedEntriesAddr()/loadedEntriesLen() post-close cannot
+            // dereference freed native memory; the getters are not closed-guarded.
+            loadedEntriesAddr = 0L;
+            loadedEntriesLen = 0;
+        }
+        if (appendMapAddr != 0L) {
+            Files.munmap(appendMapAddr, appendMapCapacity, MemoryTag.MMAP_DEFAULT);
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            // The active window reserves space past the logical end. Return that tail on
+            // orderly close; after a crash open() treats the zero-filled reserve as
+            // a torn trailing chunk and truncates it to the same appendOffset.
+            if (!ff.truncate(fd, appendOffset)) {
+                LOG.warn("symbol dict {} could not trim mmap reserve to {}; recovery will "
+                                + "discard the zero-filled tail on the next open",
+                        FILE_NAME, appendOffset);
+            }
+        }
+        if (scratchAddr != 0L) {
+            Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT);
+            scratchAddr = 0L;
+            scratchCap = 0;
+        }
+        if (fd >= 0) {
+            ff.close(fd);
+        }
+    }
+
+    @TestOnly
+    public synchronized int appendMapGrowthCount() {
+        return appendMapGrowthCount;
+    }
+
+    @TestOnly
+    public synchronized long appendWriteCount() {
+        return appendWriteCount;
+    }
+
+    /**
+     * Base address of the loaded entry region -- the concatenated
+     * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a
+     * delta section carries them (chunk headers and CRCs stripped). Zero when
+     * nothing was recovered.
+     * 

+     * Construction-phase only. This hands out a raw pointer into native
+     * memory that {@link #close()} frees and nulls, with no closed-guard and no
+     * synchronization. It is safe to read only BEFORE the slot's I/O thread and
+     * any producer append start -- i.e. while the send loop is being constructed
+     * or an orphan-drain is seeding its mirror, both of which happen-before those
+     * threads. A caller that reads it from a running thread races {@code close()}
+     * and can dereference freed memory (use-after-free).
+     */
+    public long loadedEntriesAddr() {
+        return loadedEntriesAddr;
+    }
+
+    /**
+     * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for
+     * the same reason -- see {@link #loadedEntriesAddr()}.
+     */
+    public int loadedEntriesLen() {
+        return loadedEntriesLen;
+    }
+
+    /**
+     * Transfers the recovered entry buffer to the foreground send loop. The
+     * symbol count remains unchanged because it is also the append baseline;
+     * only the native-buffer ownership moves. Construction-phase only.
+     */
+    synchronized long takeLoadedEntries() {
+        long addr = loadedEntriesAddr;
+        loadedEntriesAddr = 0L;
+        loadedEntriesLen = 0;
+        return addr;
+    }
+
+    @TestOnly
+    public boolean usedMappedRecoveryInput() {
+        return mappedRecoveryInput;
+    }
+
+    /**
+     * Decodes the recovered entries directly into {@code target} in ascending-id
+     * order. This avoids materialising a cardinality-sized temporary list that
+     * the producer would immediately copy into the global dictionary.
+     * Construction-phase only; see {@link #loadedEntriesAddr()}.
+     */
+    public void addLoadedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeLoadedSymbols(target, null);
+    }
+
+    /**
+     * Materialises the loaded entries as symbol strings in ascending-id order.
+     * Retained for recovery-format tests; production decodes directly through
+     * {@link #addLoadedSymbolsTo(GlobalSymbolDictionary)}.
+     */
+    @TestOnly
+    public ObjList readLoadedSymbols() {
+        ObjList out = new ObjList<>(Math.max(size, 1));
+        decodeLoadedSymbols(null, out);
+        return out;
+    }
+
+    private void decodeLoadedSymbols(GlobalSymbolDictionary target, ObjList out) {
+        long p = loadedEntriesAddr;
+        long limit = p + loadedEntriesLen;
+        for (int i = 0; i < size && p < limit; i++) {
+            long len = 0;
+            int shift = 0;
+            while (p < limit) {
+                byte b = Unsafe.getUnsafe().getByte(p++);
+                len |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    // Bound the varint like the other decoders: a canonical length is
+                    // <= 5 bytes. open() already CRC-validated these bytes, so this is
+                    // defensive only; the p + len > limit check below rejects the
+                    // over-long run.
+                    break;
+                }
+            }
+            if (p + len > limit) {
+                break; // defensive: torn tail (should not happen past parse in open)
+            }
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + len);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
+            p += len;
+        }
+    }
+
+    /**
+     * Number of symbols the dictionary holds (highest id + 1).
+     */
+    public int size() {
+        return size;
+    }
+
+    private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen) {
+        int fd = ff.openRW(filePath);
+        if (fd < 0) {
+            LOG.warn("symbol dict {} could not be opened (rc={}); "
+                    + "falling back to full-dictionary frames (file left intact)", filePath, fd);
+            return null;
+        }
+        int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
+        boolean mappedInput = ff == FilesFacade.INSTANCE;
+        long inputAddr = 0L;
+        long entriesAddr = 0L;
+        int entriesLen = 0;
+        try {
+            if (mappedInput) {
+                inputAddr = Files.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT);
+                if (inputAddr == Files.FAILED_MMAP_ADDRESS) {
+                    inputAddr = 0L;
+                    throw new IllegalStateException("could not mmap symbol dictionary for recovery");
+                }
+            } else {
+                // Fault-injection facades retain the positioned-read path so tests
+                // can still force short reads without mapping around the seam.
+                inputAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+                long read = ff.read(fd, inputAddr, len, 0);
+                if (read != len) {
+                    throw new IllegalStateException("short read while recovering symbol dictionary "
+                            + "[expected=" + len + ", actual=" + read + ']');
+                }
+            }
+            if (Unsafe.getUnsafe().getInt(inputAddr) != FILE_MAGIC
+                    || Unsafe.getUnsafe().getByte(inputAddr + 4) != VERSION) {
+                throw new IllegalStateException("bad magic or unknown symbol dictionary version");
+            }
+            // First pass validates chunks and totals only the entry bytes that
+            // survive. Production reads through a file-backed mapping, so this
+            // pass does not allocate an anonymous whole-file copy. The second
+            // pass copies directly into one exact-sized retained allocation.
+            RecoveryScan scan = scanRecoveredChunks(inputAddr, len);
+            entriesLen = scan.entriesLen;
+            if (entriesLen > 0) {
+                entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT);
+                copyRecoveredEntries(inputAddr, scan.validLen, entriesAddr, entriesLen);
+            }
+            if (mappedInput) {
+                Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+            } else {
+                Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+            inputAddr = 0L;
+            // Drop any torn/stale trailing bytes so a LATER, shorter append cannot
+            // leave residue past its own end. The truncate result IS checked: a file
+            // we cannot trim could still expose stale post-end bytes whose
+            // (self-consistent) chunk CRC the parse would accept at a shifted
+            // position, so a failed truncate makes the file untrusted -- return null
+            // (the sender falls back to full self-sufficient frames) and, per the
+            // never-destroy contract, leave every byte on disk.
+            if (scan.validLen < len && !ff.truncate(fd, scan.validLen)) {
+                throw new IllegalStateException("could not drop torn/stale symbol dictionary tail");
+            }
+            return new PersistedSymbolDict(
+                    ff, fd, scan.validLen, scan.count, entriesAddr, entriesLen, mappedInput);
+        } catch (Throwable t) {
+            if (inputAddr != 0L) {
+                if (mappedInput) {
+                    Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+                } else {
+                    Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+                }
+            }
+            if (entriesAddr != 0L) {
+                Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT);
+            }
+            if (fd >= 0) {
+                ff.close(fd);
+            }
+            // Pass the throwable as a trailing argument with no matching placeholder so
+            // slf4j prints the stack trace: this WARN is the only forensic record of why
+            // a load-bearing dictionary was abandoned.
+            LOG.warn("symbol dict {} recovery failed; falling back to full-dictionary frames "
+                    + "(file left intact)", filePath, t);
+            return null;
+        }
+    }
+
+    private static void copyRecoveredEntries(long inputAddr, int validLen, long entriesAddr, int entriesLen) {
+        Varint v = new Varint();
+        int diskPos = HEADER_SIZE;
+        int dstPos = 0;
+        while (diskPos < validLen) {
+            if (!v.decode(inputAddr, diskPos, validLen)
+                    || !v.decode(inputAddr, v.end, validLen)) {
+                throw new IllegalStateException("validated symbol dictionary chunk could not be decoded");
+            }
+            int entryBytes = (int) v.value;
+            int entriesStart = v.end;
+            Unsafe.getUnsafe().copyMemory(inputAddr + entriesStart, entriesAddr + dstPos, entryBytes);
+            dstPos += entryBytes;
+            diskPos = entriesStart + entryBytes + CRC_SIZE;
+        }
+        assert dstPos == entriesLen;
+    }
+
+    private static int[] buildCrc32cTable() {
+        int[] table = new int[256];
+        for (int n = 0; n < 256; n++) {
+            int c = n;
+            for (int k = 0; k < 8; k++) {
+                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
+            }
+            table[n] = c;
+        }
+        return table;
+    }
+
+    private static int crc32cRecovery(long addr, long len) {
+        int crc = ~Crc32c.INIT;
+        for (long i = 0; i < len; i++) {
+            crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF];
+        }
+        return ~crc;
+    }
+
+    private static RecoveryScan scanRecoveredChunks(long inputAddr, int len) {
+        Varint v = new Varint();
+        int count = 0;
+        int diskPos = HEADER_SIZE;
+        long entriesLen = 0L;
+        while (diskPos < len) {
+            if (!v.decode(inputAddr, diskPos, len)) {
+                break;
+            }
+            long entryCount = v.value;
+            if (!v.decode(inputAddr, v.end, len)) {
+                break;
+            }
+            long entryBytes = v.value;
+            int entriesStart = v.end;
+            long chunkEnd = (long) entriesStart + entryBytes;
+            if (chunkEnd + CRC_SIZE > len) {
+                break;
+            }
+            int chunkEndI = (int) chunkEnd;
+            int crcStored = Unsafe.getUnsafe().getInt(inputAddr + chunkEndI);
+            int crcCalc = crc32cRecovery(inputAddr + diskPos, chunkEndI - diskPos);
+            if (crcCalc != crcStored
+                    || entryCount <= 0
+                    || (long) count + entryCount > Integer.MAX_VALUE
+                    || entriesLen + entryBytes > Integer.MAX_VALUE) {
+                break;
+            }
+            entriesLen += entryBytes;
+            diskPos = chunkEndI + CRC_SIZE;
+            count += (int) entryCount;
+        }
+        return new RecoveryScan(count, (int) entriesLen, diskPos);
+    }
+
+    private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
+        int fd = ff.openCleanRW(filePath);
+        if (fd < 0) {
+            LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd);
+            return null;
+        }
+        long hdr = 0L;
+        try {
+            hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC);
+            Unsafe.getUnsafe().putByte(hdr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0);
+            long written = ff.write(fd, hdr, HEADER_SIZE, 0);
+            if (written != HEADER_SIZE) {
+                int fdToClose = fd;
+                fd = -1; // relinquish before close so the catch cannot double-close if close throws
+                ff.close(fdToClose);
+                ff.remove(filePath); // drop the headerless stub rather than litter
+                LOG.warn("symbol dict {} header write failed; proceeding without it", filePath);
+                return null;
+            }
+        } catch (Throwable t) {
+            // Unreachable with FilesFacade.INSTANCE (Files.write is native and returns
+            // -1 rather than throwing; the Unsafe puts target a valid 8-byte buffer and
+            // an 8-byte malloc cannot realistically OOM), but the ff seam exists so
+            // tests CAN inject a throwing facade -- close the fd and drop the stub so
+            // neither leaks.
+            if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing
+                int fdToClose = fd;
+                fd = -1;
+                ff.close(fdToClose);
+                ff.remove(filePath);
+            }
+            LOG.warn("symbol dict {} creation failed; proceeding without it", filePath, t);
+            return null;
+        } finally {
+            if (hdr != 0L) {
+                Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            }
+        }
+        return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0, false);
+    }
+
+    /**
+     * Verifies that {@code count} wire entries ({@code [len varint][utf8]}) occupy
+     * exactly {@code len} bytes from {@code addr}. Returns {@code true} when the triple
+     * is consistent; throws {@link IllegalStateException} (naming the offending entry /
+     * count / consumed bytes) otherwise. Called only from an {@code assert} in
+     * {@link #appendRawEntries}: it guards an internal invariant the sole caller cannot
+     * violate today, so it runs under the test suite's {@code -ea} but is elided from
+     * the per-flush production path (client apps run without {@code -ea}).
+     */
+    private static boolean validateRawEntries(long addr, int len, int count) {
+        long src = addr;
+        long srcLimit = addr + len;
+        for (int i = 0; i < count; i++) {
+            long symLen = 0;
+            int shift = 0;
+            while (src < srcLimit) {
+                byte b = Unsafe.getUnsafe().getByte(src++);
+                symLen |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    // A canonical entry-length varint is <= 5 bytes; a longer
+                    // continuation run is corrupt. The bound check below rejects it.
+                    break;
+                }
+            }
+            src += symLen; // src was just past the len varint
+            if (src > srcLimit) {
+                throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME
+                        + " [entry=" + i + ", count=" + count + ']');
+            }
+        }
+        if (src != srcLimit) {
+            throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to "
+                    + FILE_NAME + " [count=" + count + ", len=" + len
+                    + ", consumed=" + (int) (src - addr) + ']');
+        }
+        return true;
+    }
+
+    private long checkedRequiredOffset(long recordLen) {
+        long required = appendOffset + recordLen;
+        if (recordLen < 0L || required < appendOffset) {
+            throw new IllegalStateException("symbol dict append offset overflow to "
+                    + FILE_NAME + " [offset=" + appendOffset + ", recordLen=" + recordLen + ']');
+        }
+        return required;
+    }
+
+    /**
+     * Commits a chunk already assembled directly in the append mmap. The logical
+     * offset and symbol count advance last, after the CRC and a store fence, so an
+     * interrupted process leaves either a complete chunk or a tail that open()
+     * rejects and truncates.
+     */
+    private void commitMappedChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        long bodyLen = (long) hdrLen + entriesLen;
+        long recLen = bodyLen + CRC_SIZE;
+        Unsafe.getUnsafe().putInt(
+                recStart + bodyLen,
+                Crc32c.update(Crc32c.INIT, recStart, bodyLen));
+        Unsafe.getUnsafe().storeFence();
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Ensures the production append mmap covers the absolute file offset
+     * {@code required}. The log is mapped in 64 KiB-aligned windows: small flushes
+     * share a window, while a large existing dictionary does not force the process
+     * to reserve and remap its whole prefix (or geometrically over-allocate it).
+     */
+    private void ensureAppendMap(long required) {
+        if (appendMapAddr != 0L
+                && required >= appendMapOffset
+                && required <= appendMapOffset + appendMapCapacity) {
+            return;
+        }
+        long newOffset = appendOffset - appendOffset % INITIAL_APPEND_MAP_CAPACITY;
+        long needed = required - newOffset;
+        long newCapacity = Math.max(INITIAL_APPEND_MAP_CAPACITY, needed);
+        long remainder = newCapacity % INITIAL_APPEND_MAP_CAPACITY;
+        if (remainder != 0L) {
+            long padding = INITIAL_APPEND_MAP_CAPACITY - remainder;
+            if (newCapacity > Long.MAX_VALUE - padding) {
+                throw new IllegalStateException("symbol dict mmap capacity overflow to "
+                        + FILE_NAME + " [required=" + required + ']');
+            }
+            newCapacity += padding;
+        }
+        if (newOffset > Long.MAX_VALUE - newCapacity) {
+            throw new IllegalStateException("symbol dict mmap file offset overflow to "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        long newFileSize = newOffset + newCapacity;
+        if (!ff.allocate(fd, newFileSize)) {
+            throw new IllegalStateException("could not grow mmap append region for "
+                    + FILE_NAME + " [required=" + required + ", fileSize=" + newFileSize + ']');
+        }
+        if (appendMapAddr != 0L) {
+            long oldAddr = appendMapAddr;
+            long oldCapacity = appendMapCapacity;
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            Files.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT);
+        }
+        long newAddr = Files.mmap(
+                fd, newCapacity, newOffset, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
+        if (newAddr == Files.FAILED_MMAP_ADDRESS) {
+            throw new IllegalStateException("could not mmap append region for "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        appendMapAddr = newAddr;
+        appendMapCapacity = newCapacity;
+        appendMapOffset = newOffset;
+        appendMapGrowthCount++;
+    }
+
+    /**
+     * Grows the append scratch buffer to hold at least {@code required} bytes.
+     * 

+     * Throws when {@code required} exceeds {@link #MAX_SCRATCH_BYTES} rather than
+     * clamping to it: a clamp would hand back a buffer SMALLER than the caller asked
+     * for and return normally, and every caller then writes {@code required} bytes
+     * into it -- turning a clean out-of-memory into a silent native-heap overflow, on
+     * the very write path the dictionary's integrity rests on. This is the same
+     * loud-failure shape {@code CursorWebSocketSendLoop.ensureSentDictCapacity} uses
+     * on the same bound. Unreachable at any realistic cardinality (it needs a ~2 GiB
+     * dictionary section in a single frame, itself bounded by the server's batch
+     * cap), but a size guard on a data-integrity write path must never
+     * under-allocate.
+     */
+    private void ensureScratch(long required) {
+        if (scratchCap >= required) {
+            return;
+        }
+        if (required > MAX_SCRATCH_BYTES) {
+            throw new IllegalStateException("symbol dict scratch buffer exceeds the maximum size to "
+                    + FILE_NAME + " [required=" + required + ", max=" + MAX_SCRATCH_BYTES + ']');
+        }
+        // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and
+        // would make the realloc size negative.
+        long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2));
+        if (newCap > MAX_SCRATCH_BYTES) {
+            newCap = MAX_SCRATCH_BYTES;
+        }
+        scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+        scratchCap = (int) newCap;
+    }
+
+    /**
+     * Checksums {@code [recStart, recStart + hdrLen + entriesLen)} in ONE native
+     * call, appends the CRC, and issues ONE positioned write. Advances
+     * {@code size}/{@code appendOffset} only on a complete write, so a short write
+     * throws and a retry keyed off {@link #size()} re-persists the same range at the
+     * same offset.
+     */
+    private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        int bodyLen = hdrLen + entriesLen;
+        int recLen = bodyLen + CRC_SIZE;
+        Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen));
+        appendWriteCount++;
+        long written = ff.write(fd, recStart, recLen, appendOffset);
+        if (written != recLen) {
+            throw new IllegalStateException("short write to " + FILE_NAME
+                    + " [expected=" + recLen + ", actual=" + written + ']');
+        }
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Writes one chunk whose entry region is ALREADY encoded in scratch at offset
+     * {@link #MAX_CHUNK_HEADER_SIZE}. Back-fills the header immediately in front of
+     * the entries -- the header is at most {@code MAX_CHUNK_HEADER_SIZE} bytes, so it
+     * always fits the reserve -- leaving header, entries and CRC one contiguous run
+     * for a single checksum and a single write.
+     */
+    private void writeChunkFromScratch(int entriesLen, int count) {
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(entriesLen);
+        long recStart = scratchAddr + MAX_CHUNK_HEADER_SIZE - hdrLen;
+        long p = NativeBufferWriter.writeVarint(recStart, count);
+        NativeBufferWriter.writeVarint(p, entriesLen);
+        flushChunk(recStart, hdrLen, entriesLen, count);
+    }
+
+    private static final class RecoveryScan {
+        private final int count;
+        private final int entriesLen;
+        private final int validLen;
+
+        private RecoveryScan(int count, int entriesLen, int validLen) {
+            this.count = count;
+            this.entriesLen = entriesLen;
+            this.validLen = validLen;
+        }
+    }
+
+    /**
+     * Zero-allocation LEB128 decoder, one instance per recovery pass -- not one
+     * per chunk. The previous {@code long[]}-returning decoder allocated once per
+     * entry, so a million-symbol recovery churned a million throwaway arrays.
+     */
+    private static final class Varint {
+        int end;
+        long value;
+
+        /**
+         * Decodes the varint at {@code buf[pos..limit)}. Returns false -- leaving
+         * {@code value}/{@code end} undefined -- when the varint is truncated (a torn
+         * tail) or runs longer than a canonical 5-byte length.
+         */
+        boolean decode(long buf, int pos, int limit) {
+            long v = 0;
+            int shift = 0;
+            int cur = pos;
+            while (cur < limit) {
+                byte b = Unsafe.getUnsafe().getByte(buf + cur);
+                cur++;
+                v |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    value = v;
+                    end = cur;
+                    return true;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    return false; // implausible for a chunk header; treat as torn
+                }
+            }
+            return false;
+        }
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
new file mode 100644
index 00000000..e94e68f5
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -0,0 +1,308 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+
+/**
+ * Engine-owned result of the one ordered frame walk performed during recovery.
+ * It folds commit-boundary detection, delta extrema, gap detection and the raw
+ * symbol suffix into the same pass. Running state is checkpointed only at a
+ * commit-bearing frame, so a deferred orphan tail can be scanned without ever
+ * leaking its metadata or symbols into the committed result.
+ */
+final class RecoveredFrameAnalysis implements QuietCloseable {
+
+    private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8;
+    private final long ackedFsn;
+    private final int baseline;
+    private long committedBoundaryFsn = -1L;
+    private long committedCoverage;
+    private boolean committedGap;
+    private long committedMaxDeltaEnd;
+    private long committedMaxDeltaStart;
+    private int committedRawCount;
+    private int committedRawLen;
+    private long framesVisited;
+    private boolean runningGap;
+    private boolean runningUnackedGap;
+    private long runningMaxDeltaEnd;
+    private long runningMaxDeltaStart;
+    private long symbolEntriesVisited;
+    private long rawAddr;
+    private int rawCapacity;
+    private int runningRawCount;
+    private int runningRawLen;
+    private long runningCoverage;
+
+    RecoveredFrameAnalysis(int baseline, long ackedFsn) {
+        this.baseline = baseline;
+        this.ackedFsn = ackedFsn;
+        this.runningCoverage = baseline;
+        this.committedCoverage = baseline;
+    }
+
+    void accept(long fsn, long payload, int payloadLen) {
+        framesVisited++;
+        boolean isQwp = payloadLen >= QwpConstants.HEADER_SIZE
+                && payloadLen > QwpConstants.HEADER_OFFSET_FLAGS
+                && Unsafe.getUnsafe().getInt(payload) == QwpConstants.MAGIC_MESSAGE;
+        byte flags = isQwp
+                ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS)
+                : 0;
+        if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) {
+            foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
+        }
+
+        // Only a positively identified deferred QWP frame can belong to an
+        // uncommitted tail. Short, foreign or otherwise non-QWP payloads remain
+        // retirement barriers, matching findLastFrameFsnWithoutPayloadFlag.
+        if (!isQwp || (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0) {
+            committedBoundaryFsn = fsn;
+            committedCoverage = runningCoverage;
+            committedGap = runningGap;
+            committedMaxDeltaEnd = runningMaxDeltaEnd;
+            committedMaxDeltaStart = runningMaxDeltaStart;
+            committedRawLen = runningRawLen;
+            committedRawCount = runningRawCount;
+        }
+    }
+
+    void appendDecodedSymbols(ObjList out) {
+        decodeSymbols(null, out);
+    }
+
+    void addDecodedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeSymbols(target, null);
+    }
+
+    private void decodeSymbols(GlobalSymbolDictionary target, ObjList out) {
+        long p = rawAddr;
+        long limit = rawAddr + committedRawLen;
+        for (int i = 0; i < committedRawCount; i++) {
+            long encoded = readVarint(p, limit);
+            if (encoded < 0L) {
+                throw new IllegalStateException("malformed cached symbol dictionary suffix");
+            }
+            int varintLen = (int) (encoded & 7L);
+            long symbolLen = encoded >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                throw new IllegalStateException("truncated cached symbol dictionary suffix");
+            }
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + symbolLen);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
+            p += symbolLen;
+        }
+        if (p != limit) {
+            throw new IllegalStateException("overfilled cached symbol dictionary suffix");
+        }
+    }
+
+    int baseline() {
+        return baseline;
+    }
+
+    long commitBoundaryFsn() {
+        return committedBoundaryFsn;
+    }
+
+    long coverage() {
+        return committedGap ? -1L : committedCoverage;
+    }
+
+    long framesVisited() {
+        return framesVisited;
+    }
+
+    long maxDeltaEnd() {
+        return committedMaxDeltaEnd;
+    }
+
+    long maxDeltaStart() {
+        return committedMaxDeltaStart;
+    }
+
+    long rawAddr() {
+        return rawAddr;
+    }
+
+    int rawCount() {
+        return committedRawCount;
+    }
+
+    int rawLen() {
+        return committedRawLen;
+    }
+
+    long symbolEntriesVisited() {
+        return symbolEntriesVisited;
+    }
+
+    @Override
+    public void close() {
+        if (rawAddr != 0L) {
+            Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawAddr = 0L;
+            rawCapacity = 0;
+            runningRawLen = 0;
+            committedRawLen = 0;
+        }
+    }
+
+    private void appendRaw(long entryStart, int entryLen) {
+        long required = (long) runningRawLen + entryLen;
+        if (required > MAX_RAW_BYTES) {
+            throw new IllegalStateException("recovered symbol dictionary suffix exceeds maximum size "
+                    + "[required=" + required + ", max=" + MAX_RAW_BYTES + ']');
+        }
+        if (required > rawCapacity) {
+            long newCapacity = Math.max(required, Math.max(4_096L, (long) rawCapacity * 2L));
+            if (newCapacity > MAX_RAW_BYTES) {
+                newCapacity = MAX_RAW_BYTES;
+            }
+            rawAddr = Unsafe.realloc(rawAddr, rawCapacity, (int) newCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawCapacity = (int) newCapacity;
+        }
+        Unsafe.getUnsafe().copyMemory(entryStart, rawAddr + runningRawLen, entryLen);
+        runningRawLen += entryLen;
+        runningRawCount++;
+    }
+
+    private void foldDelta(long fsn, long p, long limit) {
+        long encodedStart = readVarint(p, limit);
+        if (encodedStart < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int startLen = (int) (encodedStart & 7L);
+        long deltaStart = encodedStart >>> 3;
+        p += startLen;
+        if (deltaStart > runningMaxDeltaStart) {
+            runningMaxDeltaStart = deltaStart;
+        }
+
+        long encodedCount = readVarint(p, limit);
+        if (encodedCount < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int countLen = (int) (encodedCount & 7L);
+        long deltaCount = encodedCount >>> 3;
+        p += countLen;
+        long deltaEnd = deltaCount > Long.MAX_VALUE - deltaStart
+                ? Long.MAX_VALUE
+                : deltaStart + deltaCount;
+        if (deltaEnd > runningMaxDeltaEnd) {
+            runningMaxDeltaEnd = deltaEnd;
+        }
+        if (runningGap) {
+            // A full dictionary is a new self-sufficient epoch, but it may only
+            // repair a gap that is entirely behind the durable ACK watermark.
+            // If any gapped frame will replay first, accepting this reset would
+            // hide the unsafe wire-order gap and let the server observe missing
+            // ids before it reaches the full frame.
+            if (deltaStart != 0L || runningUnackedGap) {
+                return;
+            }
+            runningGap = false;
+            runningCoverage = baseline;
+            runningRawLen = 0;
+            runningRawCount = 0;
+        }
+        if (deltaStart > runningCoverage) {
+            markGap(fsn);
+            return;
+        }
+        // The segment scan already CRC-validated this frame. When its entire
+        // range is covered, entry parsing cannot extend recovery state, so skip
+        // the cardinality-proportional payload walk.
+        if (deltaEnd <= runningCoverage) {
+            return;
+        }
+
+        long id = deltaStart;
+        for (long i = 0; i < deltaCount; i++, id++) {
+            symbolEntriesVisited++;
+            long entryStart = p;
+            long encodedLen = readVarint(p, limit);
+            if (encodedLen < 0L) {
+                markGap(fsn);
+                return;
+            }
+            int varintLen = (int) (encodedLen & 7L);
+            long symbolLen = encodedLen >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                markGap(fsn);
+                return;
+            }
+            p += symbolLen;
+            if (id >= runningCoverage) {
+                appendRaw(entryStart, (int) (p - entryStart));
+            }
+        }
+        if (deltaEnd > runningCoverage) {
+            runningCoverage = deltaEnd;
+        }
+    }
+
+    private void markGap(long fsn) {
+        runningGap = true;
+        if (fsn > ackedFsn) {
+            runningUnackedGap = true;
+        }
+    }
+
+    /**
+     * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an
+     * unterminated/oversized 32-bit protocol varint.
+     */
+    private static long readVarint(long p, long limit) {
+        long value = 0L;
+        int shift = 0;
+        int bytes = 0;
+        while (p < limit && bytes < 5) {
+            byte b = Unsafe.getUnsafe().getByte(p++);
+            value |= (long) (b & 0x7F) << shift;
+            bytes++;
+            if ((b & 0x80) == 0) {
+                return (value << 3) | bytes;
+            }
+            shift += 7;
+        }
+        return -1L;
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index c8516c4c..5ba787f3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -62,6 +62,8 @@ public final class SegmentRing implements QuietCloseable {
     public static final long BACKPRESSURE_NO_SPARE = -1L;
     /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */
     public static final long PAYLOAD_TOO_LARGE = -2L;
+    static final String LEGACY_READER_GUARD_A = ".qwp-v2-guard-a.sfa";
+    static final String LEGACY_READER_GUARD_B = ".qwp-v2-guard-b.sfa";
     private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class);
     // Tally of baseSeq comparisons performed by sortByBaseSeq across every
     // openExisting() recovery on this JVM. Used by SegmentRingTest to
@@ -178,7 +180,9 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                 int rc = 1;
                 while (rc > 0) {
                     String name = Files.utf8ToString(Files.findName(find));
-                    if (name != null && name.endsWith(".sfa")) {
+                    if (name != null
+                            && name.endsWith(".sfa")
+                            && !isLegacyReaderGuard(name)) {
                         String path = sfDir + "/" + name;
                         MmapSegment seg = null;
                         try {
@@ -325,6 +329,23 @@ public long ackedFsn() {
         return ackedFsn;
     }
 
+    /**
+     * Installs an on-disk rollback barrier before any v2 segment can be
+     * created or appended. The guards are valid v1 segments so a legacy
+     * reader cannot skip them as an unknown format. Their deliberately
+     * non-contiguous FSN ranges make its recovery fail closed before replay;
+     * current readers recognize the reserved names and omit them from the
+     * logical ring.
+     */
+    static void installLegacyReaderBarrier(String sfDir) {
+        MmapSegment.createLegacyReaderGuard(
+                sfDir + '/' + LEGACY_READER_GUARD_A,
+                Long.MAX_VALUE - 4L);
+        MmapSegment.createLegacyReaderGuard(
+                sfDir + '/' + LEGACY_READER_GUARD_B,
+                Long.MAX_VALUE - 1L);
+    }
+
     /**
      * I/O thread (or anyone tracking ACK) advances the ACK cursor. {@code seq}
      * is cumulative -- the server has confirmed every FSN up to and including
@@ -546,6 +567,98 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag
         return Math.max(best, fsn);
     }
 
+    /**
+     * Rebuilds the symbols a replay of {@code [minFsnInclusive .. maxFsnInclusive]} would
+     * register ABOVE {@code baseline}, appending them to {@code out} in ascending id order.
+     * Returns the coverage the replay establishes, or {@code -1} on a genuine gap. See
+     * {@link MmapSegment#collectReplaySymbolsAbove}.
+     * 

+     * Walks the segments in ascending FSN order -- {@code sealedSegments} are held in
+     * baseSeq order and {@code active} is the newest (see {@code recover}) -- threading the
+     * coverage through, because unlike {@link #maxSymbolDeltaEnd} (a plain max) this walk is
+     * ORDER-DEPENDENT: each frame may only extend a dictionary the frames before it built.
+     */
+    public synchronized long collectReplaySymbolsAbove(
+            int headerMagic,
+            int flagsOffset,
+            int flagDeltaMask,
+            int qwpHeaderSize,
+            long minFsnInclusive,
+            long maxFsnInclusive,
+            long baseline,
+            ObjList out
+    ) {
+        long coverage = baseline;
+        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+            coverage = sealedSegments.get(i).collectReplaySymbolsAbove(
+                    headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize,
+                    minFsnInclusive, maxFsnInclusive, coverage, out);
+            if (coverage < 0) {
+                return -1L;
+            }
+        }
+        return active.collectReplaySymbolsAbove(
+                headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize,
+                minFsnInclusive, maxFsnInclusive, coverage, out);
+    }
+
+    /**
+     * Performs the one ordered recovery fold across sealed segments and the
+     * active segment. The returned native suffix remains owned by the caller.
+     */
+    synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) {
+        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline, ackedFsn);
+        try {
+            for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+                sealedSegments.get(i).scanRecovery(analysis);
+            }
+            active.scanRecovery(analysis);
+            return analysis;
+        } catch (Throwable t) {
+            analysis.close();
+            throw t;
+        }
+    }
+
+    /**
+     * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below
+     * {@code maxFsnInclusive} references (0 when none). See
+     * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a
+     * persisted dictionary torn below the ids the surviving committed frames
+     * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred
+     * tail and are excluded.
+     */
+    public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) {
+        long maxEnd = 0L;
+        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+            long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive);
+            if (end > maxEnd) {
+                maxEnd = end;
+            }
+        }
+        long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive);
+        return Math.max(maxEnd, end);
+    }
+
+    /**
+     * Highest {@code deltaStart} any symbol-dict delta frame at or below
+     * {@code maxFsnInclusive} carries (0 when none, or when every such frame is
+     * self-sufficient). See {@link MmapSegment#maxSymbolDeltaStart}; paired with
+     * {@link #maxSymbolDeltaEnd} at recovery to tell a full-dict-fallback slot
+     * (recoverable with no dictionary) from a torn delta dictionary (fails clean).
+     */
+    public synchronized long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) {
+        long maxStart = 0L;
+        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+            long start = sealedSegments.get(i).maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive);
+            if (start > maxStart) {
+                maxStart = start;
+            }
+        }
+        long start = active.maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive);
+        return Math.max(maxStart, start);
+    }
+
     public MmapSegment getActive() {
         return active;
     }
@@ -767,6 +880,10 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) {
         }
     }
 
+    private static boolean isLegacyReaderGuard(String name) {
+        return LEGACY_READER_GUARD_A.equals(name) || LEGACY_READER_GUARD_B.equals(name);
+    }
+
     private static void swap(ObjList list, int i, int j) {
         if (i == j) return;
         MmapSegment tmp = list.get(i);
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
index 0b8379de..0ec9f9f5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
@@ -31,12 +31,18 @@
 import io.questdb.client.std.Unsafe;
 
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
- * Advisory exclusive lock for a single SF slot directory.
+ * Advisory exclusive locks for a single SF slot.
  * 

- * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx}
- * for the entire lifetime of the engine that owns the slot. The lock is
+ * {@link #acquire(String)} locks a {@code .lock} file inside the slot directory
+ * for the entire lifetime of the engine that owns it. {@link #acquireLogical(String)}
+ * locks a sibling file under the parent SF directory for short-lived pathname
+ * transitions and orphan adoption; because it is outside the slot directory,
+ * it remains stable if that directory is renamed. Both use
+ * {@code flock}/{@code LockFileEx}. A lock is
  * automatically released when the fd is closed — including on hard process
  * exit, since the kernel cleans up file locks for terminated processes.
  * 

@@ -58,6 +64,7 @@ public final class SlotLock implements QuietCloseable {
 
     private static final String LOCK_FILE_NAME = ".lock";
     private static final String LOCK_PID_FILE_NAME = ".lock.pid";
+    private static final String LOGICAL_LOCK_DIR_NAME = ".slot-locks";
     private final String slotDir;
     private int fd;
 
@@ -75,18 +82,44 @@ private SlotLock(String slotDir, int fd) {
      *                               or lock contention.
      */
     public static SlotLock acquire(String slotDir) {
-        if (slotDir == null || slotDir.isEmpty()) {
-            throw new IllegalArgumentException("slotDir must not be empty");
-        }
-        if (!Files.exists(slotDir)) {
-            int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT);
-            if (rc != 0) {
-                throw new IllegalStateException(
-                        "could not create slot dir: " + slotDir + " rc=" + rc);
-            }
-        }
+        validateSlotDir(slotDir);
+        ensureDirectory(slotDir, "slot dir");
         String lockPath = slotDir + "/" + LOCK_FILE_NAME;
         String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME;
+        return acquireAt(slotDir, lockPath, pidPath);
+    }
+
+    /**
+     * Acquires the stable logical lock for {@code slotDir}. Unlike the
+     * directory-local {@code .lock}, this lock is anchored in the parent SF
+     * directory, so renaming the slot cannot move the lock inode away from the
+     * logical slot name it guards.
+     * 

+     * Callers use this as a short-lived transition/adoption lock, always before
+     * acquiring the directory-local lock. In particular it must cover an
+     * unreplayable slot's close -> rename -> recreate transition, preventing a
+     * queued orphan drainer from adopting the renamed inode and later touching
+     * the fresh directory through the old pathname.
+     */
+    public static SlotLock acquireLogical(String slotDir) {
+        validateSlotDir(slotDir);
+        Path slotPath = Paths.get(slotDir);
+        Path parentPath = slotPath.getParent();
+        Path slotNamePath = slotPath.getFileName();
+        if (parentPath == null || slotNamePath == null || slotNamePath.toString().isEmpty()) {
+            throw new IllegalArgumentException(
+                    "slotDir must contain a parent and slot name: " + slotDir);
+        }
+        String parentDir = parentPath.toString();
+        String slotName = slotNamePath.toString();
+        String logicalLockDir = parentDir + "/" + LOGICAL_LOCK_DIR_NAME;
+        ensureDirectory(logicalLockDir, "logical slot lock dir");
+        String lockPath = logicalLockDir + "/" + slotName + ".lock";
+        String pidPath = logicalLockDir + "/" + slotName + ".lock.pid";
+        return acquireAt(slotDir, lockPath, pidPath);
+    }
+
+    private static SlotLock acquireAt(String slotDir, String lockPath, String pidPath) {
         int fd = Files.openRW(lockPath);
         if (fd < 0) {
             throw new IllegalStateException(
@@ -111,6 +144,25 @@ public static SlotLock acquire(String slotDir) {
         }
     }
 
+    private static void ensureDirectory(String path, String description) {
+        if (!Files.exists(path)) {
+            int rc = Files.mkdir(path, Files.DIR_MODE_DEFAULT);
+            // Multiple senders may create the shared parent lock directory
+            // concurrently. Treat EEXIST as success, just as the builder does
+            // for the SF root itself.
+            if (rc != 0 && !Files.exists(path)) {
+                throw new IllegalStateException(
+                        "could not create " + description + ": " + path + " rc=" + rc);
+            }
+        }
+    }
+
+    private static void validateSlotDir(String slotDir) {
+        if (slotDir == null || slotDir.isEmpty()) {
+            throw new IllegalArgumentException("slotDir must not be empty");
+        }
+    }
+
     /** Slot dir this lock guards. */
     public String slotDir() {
         return slotDir;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
new file mode 100644
index 00000000..00c0d351
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
@@ -0,0 +1,51 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+
+/**
+ * A recovered store-and-forward slot whose symbol dictionary cannot be rebuilt at all --
+ * neither from the persisted side-file's intact prefix nor from the surviving frames' own
+ * delta sections. Its frames reference ids that nothing still holds, so replaying them
+ * would null-pad the gap on the server and silently misattribute symbol values.
+ * 

+ * This is the ONE verdict {@code seedGlobalDictionaryFromPersisted} reaches after it has
+ * tried every source of truth it has, so it is the only signal that may quarantine a slot.
+ * A distinct type, rather than a message match, because the difference between "this slot
+ * is unreplayable" and any other {@link LineSenderException} out of the connect path is the
+ * difference between setting a slot aside and silently discarding one that was fine.
+ * 

+ * It stays a {@code LineSenderException} so that a caller which does NOT handle it -- the
+ * background drainer, a test constructing the sender directly -- keeps the old fail-clean
+ * behaviour rather than seeing a new checked type. Only {@code Sender.build()} treats it as
+ * recoverable, by setting the slot aside and starting the producer on a fresh one.
+ */
+public class UnreplayableSlotException extends LineSenderException {
+
+    public UnreplayableSlotException(CharSequence message) {
+        super(message);
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
index a255e638..7097f3cd 100644
--- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
+++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
@@ -77,6 +77,7 @@ public final class ConfigSchema {
         // (one vocabulary, side-owned application).
         str("max_frame_rejections", Side.INGRESS);
         str("poison_min_escalation_window_millis", Side.INGRESS);
+        str("catchup_cap_gap_min_escalation_window_millis", Side.INGRESS);
         str("max_name_len", Side.INGRESS);
         str("reconnect_initial_backoff_millis", Side.INGRESS);
         str("reconnect_max_backoff_millis", Side.INGRESS);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java
new file mode 100644
index 00000000..400ea1d3
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java
@@ -0,0 +1,562 @@
+/*+*****************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode).
+ * 

+ * When a memory-mode sender reconnects, the server it lands on has an empty
+ * dictionary (the server discards it on every disconnect). Because the producer
+ * ships monotonic deltas -- each symbol id once -- a naive replay would leave the
+ * fresh server with a dictionary gap. The I/O thread prevents this by sending a
+ * full-dictionary catch-up frame before any post-reconnect traffic. This test
+ * reconstructs the server's per-connection dictionary from the captured wire
+ * bytes and asserts it stays complete and gap-free across the reconnect.
+ */
+public class DeltaDictCatchUpTest {
+
+    @Test
+    public void testReconnectCatchUpRebuildsDictionary() throws Exception {
+        // Connection 1: send "alpha" (id 0), ACK it, then drop the socket so the
+        // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1).
+        // Without catch-up, connection 2's first data frame would carry
+        // deltaStart=1 and the fresh server would never learn id 0.
+        assertMemoryLeak(() -> {
+            CatchUpHandler handler = new CatchUpHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+                    sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.dictFor(1).size() >= 1, 5_000);
+
+                    // Wait until the server has actually closed connection 1 before
+                    // sending batch 2, so batch 2 cannot race into connection 1 and
+                    // must drive the reconnect + catch-up.
+                    waitFor(() -> handler.conn1Closed, 5_000);
+
+                    sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.connectionsAccepted.get() >= 2
+                            && handler.dictFor(2).size() >= 2, 5_000);
+                }
+
+                // The fresh (2nd) connection's dictionary, rebuilt purely from the
+                // frames it received, must hold both symbols contiguously with no
+                // null gap -- exactly what the catch-up frame guarantees.
+                List conn2 = handler.dictFor(2);
+                Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables",
+                        handler.sawZeroTableFrameOnConn2);
+                Assert.assertTrue("the catch-up frame carries no rows, so it must defer its "
+                                + "(empty) commit -- FLAG_DEFER_COMMIT set",
+                        handler.catchUpDeferredOnConn2);
+                Assert.assertEquals("2nd connection dictionary size", 2, conn2.size());
+                Assert.assertEquals("alpha", conn2.get(0));
+                Assert.assertEquals("beta", conn2.get(1));
+                assertAckSequencesStartAtZero(handler.ackSequenceStarts());
+            }
+        });
+    }
+
+    @Test
+    public void testReconnectPreservesMonotonicDeltaBaseline() throws Exception {
+        // Regression: the producer's sent-symbol watermark (sentMaxSymbolId) must
+        // SURVIVE a reconnect. resetSymbolDictStateForNewConnection deliberately
+        // leaves it untouched -- the I/O thread re-registers the whole dictionary via
+        // a catch-up frame before replay, so the producer keeps shipping deltas ABOVE
+        // the baseline across the wire boundary. If a regression reset it on
+        // reconnect, the first post-reconnect data frame would re-ship the whole
+        // dictionary inline (deltaStart=0), pure wasted bandwidth. The sibling
+        // testReconnectCatchUpRebuildsDictionary asserts only that the final
+        // dictionary is complete -- which a reset-then-redefine ALSO satisfies (the
+        // server tolerates the redefinition) -- so it does NOT catch that regression.
+        // This pins the baseline survival directly: connection 2's data frame must
+        // carry a delta starting at id 1 (above alpha), not 0.
+        assertMemoryLeak(() -> {
+            CatchUpHandler handler = new CatchUpHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+                    // Connection 1 registers alpha (id 0); the server ACKs and drops it.
+                    sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.dictFor(1).size() >= 1, 5_000);
+                    waitFor(() -> handler.conn1Closed, 5_000);
+
+                    // Connection 2 (fresh) registers beta (id 1). With the baseline
+                    // preserved, beta ships as a delta ABOVE id 0 (deltaStart=1).
+                    sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.connectionsAccepted.get() >= 2
+                            && handler.dictFor(2).size() >= 2, 5_000);
+                }
+
+                Assert.assertTrue("connection 2 must re-register the dictionary via a catch-up first",
+                        handler.sawZeroTableFrameOnConn2);
+                Assert.assertTrue("post-reconnect data frame must ship a delta ABOVE the surviving "
+                                + "baseline (deltaStart >= 1); a reset baseline re-ships the whole "
+                                + "dictionary from deltaStart 0",
+                        handler.conn2SawDeltaAboveBaseline);
+                assertAckSequencesStartAtZero(handler.ackSequenceStarts());
+            }
+        });
+    }
+
+    @Test
+    public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exception {
+        // Regression (homogeneous single cap): a symbol whose length sits just below
+        // the advertised cap is ACCEPTED into a data frame (messageSize <= cap) and
+        // enters the sent-dictionary mirror. On reconnect the catch-up must
+        // re-register it under the SAME cap -- the bare catch-up frame (header + two
+        // varints + the entry) is smaller than the data frame that already shipped
+        // it (which also carried the table schema + a row), so it fits.
+        //
+        // Pre-fix, the single-entry terminal used the conservative PACKING budget
+        // (cap - HEADER_SIZE - 16), which is stricter than the producer's publish
+        // gate (messageSize <= cap) by more than the minimal data-frame overhead. So
+        // a symbol accepted onto the wire under cap C could exceed that budget and
+        // trip a spurious "during catch-up" terminal, permanently hard-failing a
+        // running producer on its first transient reconnect. Concretely at cap=200:
+        // table("t").symbol("s", <173 chars>).atNow() encodes to 198 bytes (<=200,
+        // accepted), its dict entry is 2+173=175 bytes (> old budget 172 -> old
+        // terminal), while the real solo catch-up frame is 12+1+1+175=189 (<=200 ->
+        // fits). Unlike testCatchUpEntryTooLargeForCapFailsTerminally (a genuinely
+        // oversized entry on a shrunk cap, which MUST still terminate), this entry
+        // is legally shippable and must NOT terminate.
+        final int cap = 200;
+        final String nearCapSymbol = TestUtils.repeat("x", 173);
+        assertMemoryLeak(() -> {
+            CatchUpHandler handler = new CatchUpHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                server.setAdvertisedMaxBatchSize(cap); // same cap on every handshake (homogeneous)
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+                    // Symbol-only row so the near-cap symbol drives the frame size.
+                    sender.table("t").symbol("s", nearCapSymbol).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.dictFor(1).size() >= 1, 5_000);
+                    waitFor(() -> handler.conn1Closed, 5_000);
+
+                    // The reconnect runs the catch-up under the SAME cap. Pre-fix
+                    // this latched a terminal (surfacing on this flush); post-fix the
+                    // catch-up ships the near-cap symbol and the flush goes through.
+                    sender.table("t").symbol("s", "beta").atNow();
+                    sender.flush();
+                    waitFor(() -> handler.connectionsAccepted.get() >= 2
+                            && handler.dictFor(2).size() >= 2, 5_000);
+                }
+
+                // Connection 2's dictionary, rebuilt purely from the frames it
+                // received, must hold the near-cap symbol (re-registered by the
+                // catch-up) and beta, gap-free -- proving the catch-up SHIPPED rather
+                // than terminated.
+                List conn2 = handler.dictFor(2);
+                Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables",
+                        handler.sawZeroTableFrameOnConn2);
+                Assert.assertEquals("2nd connection dictionary size", 2, conn2.size());
+                Assert.assertEquals(nearCapSymbol, conn2.get(0));
+                Assert.assertEquals("beta", conn2.get(1));
+                assertAckSequencesStartAtZero(handler.ackSequenceStarts());
+            }
+        });
+    }
+
+    @Test
+    public void testForegroundCatchUpCapGapRetriesPastOrphanBudgetAndRecovers() throws Exception {
+        // A dictionary entry that exceeds the reconnect server's per-chunk budget
+        // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. A live
+        // foreground sender must keep retrying without surfacing that cluster state to
+        // the producer, even after the orphan drainer's 16-attempt quarantine budget.
+        //
+        // Connection 1 advertises no cap, so the ~202-byte symbol registers and
+        // enters the sent-dictionary mirror. The handler then shrinks the
+        // advertised cap to 160 (catch-up budget 132) and drops the socket, so the
+        // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do
+        // this: the client refuses to SEND a single-table frame over the cap, and
+        // that data frame is always larger than the bare catch-up entry.)
+        assertMemoryLeak(() -> {
+            CapShrinkHandler handler = new CapShrinkHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                handler.setServer(server);
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry
+                // Zeroing the dwell makes the old faulty foreground policy terminal at
+                // its 16th cap gap. Reaching 20 handshakes therefore proves the live loop
+                // is no longer governed by the orphan budget.
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+                        + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50"
+                        + ";catchup_cap_gap_min_escalation_window_millis=0;")) {
+                    sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow();
+                    sender.flush();
+                    waitFor(() -> server.handshakeCount() >= 20, 10_000);
+
+                    // Producer calls remain usable while the wire is stuck in the cap
+                    // gap. Then restore the larger-cap node and prove the buffered row is
+                    // actually acknowledged, not merely accepted into the local SF ring.
+                    sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow();
+                    sender.flush();
+                    server.setAdvertisedMaxBatchSize(0);
+                    sender.table("t").symbol("s", "z").longColumn("v", 3L).atNow();
+                    long targetFsn = sender.flushAndGetSequence();
+                    Assert.assertTrue("foreground sender must recover and drain after the cap is restored",
+                            sender.awaitAckedFsn(targetFsn, 5_000));
+                }
+                assertAckSequencesStartAtZero(handler.ackSequenceStarts());
+            }
+        });
+    }
+
+    @Test
+    public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception {
+        // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes),
+        // then drops once the server has learned them all. On reconnect the fresh
+        // server has an empty dictionary, so the I/O thread must replay all 40 as a
+        // catch-up -- but the server advertises a 160-byte batch cap, so the whole
+        // dictionary cannot fit in a single frame. The catch-up therefore splits
+        // into several contiguous zero-table frames that the fresh server stitches
+        // back into a complete, gap-free dictionary.
+        final int symbolCount = 40;
+        assertMemoryLeak(() -> {
+            SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount);
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+                    // One row per flush so each frame stays under the 160-byte cap; the
+                    // sent dictionary still accumulates all 40 symbols on connection 1.
+                    for (int i = 0; i < symbolCount; i++) {
+                        sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow();
+                        sender.flush();
+                    }
+                    // Wait until connection 1 has learned every symbol, so the sender's
+                    // sent-dictionary mirror (the catch-up source) holds all of them.
+                    waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000);
+
+                    // Wait until the server has actually closed connection 1 before
+                    // sending batch 2, so batch 2 drives the reconnect + split catch-up.
+                    waitFor(() -> handler.conn1Closed, 5_000);
+
+                    sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.connectionsAccepted.get() >= 2
+                            && handler.dictFor(2).size() >= symbolCount + 1, 10_000);
+                }
+
+                // Connection 2's dictionary, rebuilt purely from the frames it received,
+                // must hold every symbol contiguously with no null gap -- the split
+                // catch-up frames reassemble exactly.
+                List conn2 = handler.dictFor(2);
+                Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size());
+                for (int i = 0; i <= symbolCount; i++) {
+                    Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i));
+                }
+                // The catch-up had to span more than one zero-table frame to stay under
+                // the advertised cap -- that split is the behaviour under test.
+                Assert.assertTrue("catch-up split into multiple frames (saw "
+                                + handler.zeroTableFramesOnConn2 + ")",
+                        handler.zeroTableFramesOnConn2 >= 2);
+                assertAckSequencesStartAtZero(handler.ackSequenceStarts());
+            }
+        });
+    }
+
+    private static void assertAckSequencesStartAtZero(List ackSequenceStarts) {
+        Assert.assertFalse("the fixture must ACK at least one connection", ackSequenceStarts.isEmpty());
+        for (int i = 0; i < ackSequenceStarts.size(); i++) {
+            Assert.assertEquals("connection " + (i + 1) + " must begin ACK sequencing at zero",
+                    0L, ackSequenceStarts.get(i).longValue());
+        }
+    }
+
+    private static String symbolName(int i) {
+        // 10-char symbols so 40 of them clearly exceed the advertised 160-byte cap.
+        return "symbol" + (1000 + i);
+    }
+
+    private static void waitFor(BoolCondition cond, long timeoutMillis) {
+        long deadline = System.currentTimeMillis() + timeoutMillis;
+        while (System.currentTimeMillis() < deadline) {
+            if (cond.test()) return;
+            try {
+                Thread.sleep(20);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                Assert.fail("interrupted");
+            }
+        }
+        Assert.fail("waitFor timed out");
+    }
+
+    @FunctionalInterface
+    private interface BoolCondition {
+        boolean test();
+    }
+
+    /**
+     * Mirrors the server's per-connection delta-dictionary accumulation: the
+     * dictionary resets on every new connection, and each frame's delta section
+     * ({@code setQuick(deltaStart + i)}, null-padding to reach deltaStart) extends
+     * or overwrites it. A missing catch-up would show up here as a null gap.
+     */
+    private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler {
+        final AtomicInteger connectionsAccepted = new AtomicInteger();
+        // Set once the server has closed connection 1. A test waits on this
+        // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot
+        // race into connection 1's pre-close window and must land on the reconnect.
+        volatile boolean conn1Closed;
+        // Set from the flags byte of the zero-table catch-up frame on connection 2:
+        // the catch-up carries no rows and must defer its (empty) commit.
+        volatile boolean catchUpDeferredOnConn2;
+        // Set when connection 2 receives a DATA frame (tableCount > 0) whose delta
+        // starts ABOVE id 0 (deltaStart >= 1). This can only happen if the producer's
+        // monotonic baseline SURVIVED the reconnect: a reset would re-ship the whole
+        // dictionary from deltaStart 0. Robust to replay -- a replayed pre-reconnect
+        // frame carries its original deltaStart 0, so only a genuinely-above-baseline
+        // post-reconnect frame trips it.
+        volatile boolean conn2SawDeltaAboveBaseline;
+        volatile boolean sawZeroTableFrameOnConn2;
+        private final List ackSequenceStarts = new CopyOnWriteArrayList<>();
+        private final List> dictsByConn = new CopyOnWriteArrayList<>();
+        private TestWebSocketServer.ClientHandler currentClient;
+        private final AtomicLong nextSeq = new AtomicLong(0);
+
+        List ackSequenceStarts() {
+            return new ArrayList<>(ackSequenceStarts);
+        }
+
+        synchronized List dictFor(int connNumber) {
+            return connNumber <= dictsByConn.size()
+                    // Copy under the lock: the caller iterates it unlocked while the
+                    // server thread may still be appending to the live inner list.
+                    ? new ArrayList<>(dictsByConn.get(connNumber - 1))
+                    : new ArrayList<>();
+        }
+
+        @Override
+        public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            boolean newConnection = currentClient != client;
+            if (newConnection) {
+                currentClient = client;
+                connectionsAccepted.incrementAndGet();
+                dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection
+                nextSeq.set(0);
+            }
+            int connNumber = dictsByConn.size();
+            List dict = dictsByConn.get(connNumber - 1);
+            QwpWireTestUtils.accumulateDeltaDictionary(data, dict);
+            if (connNumber == 2) {
+                if (QwpWireTestUtils.tableCount(data) == 0) {
+                    sawZeroTableFrameOnConn2 = true;
+                    // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5).
+                    catchUpDeferredOnConn2 = (data[5] & 0x01) != 0;
+                } else if (data.length >= 12 && (data[5] & 0x08) != 0) {
+                    // A post-reconnect DATA frame carrying a delta section
+                    // (FLAG_DELTA_SYMBOL_DICT = 0x08). A deltaStart >= 1 means the
+                    // producer resumed the delta ABOVE the surviving baseline; a reset
+                    // baseline would re-ship from deltaStart 0. Checking any frame (not
+                    // just the first) keeps this robust to a replayed pre-reconnect
+                    // frame arriving ahead of the new one -- that replay carries its
+                    // original deltaStart 0 and does not trip the flag.
+                    int[] pos = {12};
+                    if (QwpWireTestUtils.readVarint(data, pos) >= 1) {
+                        conn2SawDeltaAboveBaseline = true;
+                    }
+                }
+            }
+            try {
+                long ackSequence = nextSeq.getAndIncrement();
+                if (newConnection) {
+                    ackSequenceStarts.add(ackSequence);
+                }
+                client.sendBinary(QwpWireTestUtils.buildAck(ackSequence));
+                // Drop the first connection right after ACKing its only frame,
+                // forcing the sender to reconnect onto a fresh dictionary.
+                if (connNumber == 1) {
+                    Thread.sleep(50);
+                    client.close();
+                    conn1Closed = true;
+                }
+            } catch (IOException | InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            }
+        }
+
+    }
+
+    /**
+     * ACKs connection 1's frames with no advertised cap (so an oversized symbol
+     * registers), then -- once connection 1 has sent something -- shrinks the
+     * advertised batch cap and drops the socket. The reconnect (connection 2)
+     * therefore advertises a cap whose catch-up budget is too small for the
+     * symbol, exercising the foreground retry path in sendDictCatchUp.
+     */
+    private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler {
+        final AtomicInteger connectionsAccepted = new AtomicInteger();
+        private final List ackSequenceStarts = new CopyOnWriteArrayList<>();
+        private final AtomicLong nextSeq = new AtomicLong(0);
+        private TestWebSocketServer.ClientHandler currentClient;
+        private volatile TestWebSocketServer server;
+
+        List ackSequenceStarts() {
+            return new ArrayList<>(ackSequenceStarts);
+        }
+
+        void setServer(TestWebSocketServer server) {
+            this.server = server;
+        }
+
+        @Override
+        public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            boolean newConnection = currentClient != client;
+            if (newConnection) {
+                currentClient = client;
+                connectionsAccepted.incrementAndGet();
+                nextSeq.set(0);
+            }
+            try {
+                long ackSequence = nextSeq.getAndIncrement();
+                if (newConnection) {
+                    ackSequenceStarts.add(ackSequence);
+                }
+                client.sendBinary(QwpWireTestUtils.buildAck(ackSequence));
+                if (connectionsAccepted.get() == 1) {
+                    // Connection 1 registered the big symbol. Shrink the cap so the
+                    // reconnect's catch-up budget can't fit it, then drop to force
+                    // the reconnect. Setting the cap before the close (not from the
+                    // test thread after it) removes the set-vs-reconnect race.
+                    server.setAdvertisedMaxBatchSize(160); // catch-up budget = 132
+                    Thread.sleep(50);
+                    client.close();
+                }
+            } catch (IOException | InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    /**
+     * Like {@link CatchUpHandler}, but drops connection 1 only after it has
+     * learned the whole batch, and counts the zero-table catch-up frames on
+     * connection 2 so a test can assert the dictionary replay split across
+     * several frames to respect the advertised batch cap.
+     */
+    private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler {
+        final AtomicInteger connectionsAccepted = new AtomicInteger();
+        // Set once the server has closed connection 1 (see CatchUpHandler.conn1Closed).
+        volatile boolean conn1Closed;
+        volatile int zeroTableFramesOnConn2;
+        private final List ackSequenceStarts = new CopyOnWriteArrayList<>();
+        private final List> dictsByConn = new CopyOnWriteArrayList<>();
+        private final int dropConn1AtDictSize;
+        private final AtomicLong nextSeq = new AtomicLong(0);
+        private boolean conn1Dropped;
+        private TestWebSocketServer.ClientHandler currentClient;
+
+        SplitCatchUpHandler(int dropConn1AtDictSize) {
+            this.dropConn1AtDictSize = dropConn1AtDictSize;
+        }
+
+        List ackSequenceStarts() {
+            return new ArrayList<>(ackSequenceStarts);
+        }
+
+        synchronized List dictFor(int connNumber) {
+            return connNumber <= dictsByConn.size()
+                    // Copy under the lock: the caller iterates it unlocked while the
+                    // server thread may still be appending to the live inner list.
+                    ? new ArrayList<>(dictsByConn.get(connNumber - 1))
+                    : new ArrayList<>();
+        }
+
+        @Override
+        public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            boolean newConnection = currentClient != client;
+            if (newConnection) {
+                currentClient = client;
+                connectionsAccepted.incrementAndGet();
+                dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection
+                nextSeq.set(0);
+            }
+            int connNumber = dictsByConn.size();
+            List dict = dictsByConn.get(connNumber - 1);
+            QwpWireTestUtils.accumulateDeltaDictionary(data, dict);
+            if (connNumber == 2 && QwpWireTestUtils.tableCount(data) == 0) {
+                zeroTableFramesOnConn2++;
+            }
+            try {
+                long ackSequence = nextSeq.getAndIncrement();
+                if (newConnection) {
+                    ackSequenceStarts.add(ackSequence);
+                }
+                client.sendBinary(QwpWireTestUtils.buildAck(ackSequence));
+                // Drop connection 1 only once it has learned the entire batch, so
+                // the sender's sent-dictionary mirror is complete and the reconnect
+                // catch-up must replay a dictionary larger than the batch cap.
+                if (connNumber == 1 && !conn1Dropped && dict.size() >= dropConn1AtDictSize) {
+                    conn1Dropped = true;
+                    Thread.sleep(50);
+                    client.close();
+                    conn1Closed = true;
+                }
+            } catch (IOException | InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
new file mode 100644
index 00000000..10d4ef32
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
@@ -0,0 +1,1563 @@
+/*+*****************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.DelegatingFilesFacade;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * End-to-end recovery for the delta symbol dictionary in store-and-forward mode.
+ * 

+ * A file-mode sender writes delta-encoded SYMBOL frames (each frame carries only
+ * the ids it introduces) to a slot but never drains it -- simulating a crash. A
+ * fresh sender then recovers the slot and replays those non-self-sufficient
+ * frames to a brand-new server whose dictionary starts empty. Correctness hinges
+ * on the persisted {@code .symbol-dict}: the recovering sender loads it, the I/O
+ * thread re-registers the whole dictionary via a catch-up frame, and only then do
+ * the delta frames replay. This test reconstructs the server-side dictionary from
+ * the wire and asserts it comes out complete and gap-free.
+ */
+public class DeltaDictRecoveryTest {
+
+    private static final int DISTINCT_SYMBOLS = 8;
+    private static final int ROWS = 40;
+    private String sfDir;
+
+    @Before
+    public void setUp() {
+        sfDir = Paths.get(System.getProperty("java.io.tmpdir"),
+                "qdb-delta-recov-" + System.nanoTime()).toString();
+    }
+
+    @After
+    public void tearDown() {
+        TestUtils.removeTmpDirRec(sfDir);
+    }
+
+    @Test
+    public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception {
+        assertMemoryLeak(() -> {
+            // Phase 1: silent server (no acks). Sender 1 writes symbol rows and
+            // close-fast (no drain), leaving unacked delta frames + a persisted
+            // dictionary in the slot.
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+
+                String pad = TestUtils.repeat("x", 64);
+                String cfg = "ws::addr=localhost:" + port
+                        + ";sf_dir=" + sfDir
+                        + ";sf_max_bytes=4096"
+                        + ";close_flush_timeout_millis=0;";
+                try (Sender s1 = Sender.fromConfig(cfg)) {
+                    for (int i = 0; i < ROWS; i++) {
+                        s1.table("m")
+                                .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS))
+                                .stringColumn("p", pad)
+                                .longColumn("v", i)
+                                .atNow();
+                        s1.flush();
+                    }
+                }
+            }
+
+            // Ack a prefix so recovery does NOT replay from the self-sufficient head.
+            // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the
+            // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN
+            // DISTINCT_SYMBOLS onward -- frames whose delta starts at
+            // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle
+            // reuse existing ids). The early ids those frames reference then exist
+            // ONLY in the persisted dictionary, so the reconstructed dictionary below
+            // is complete solely because the catch-up frame re-registered them. That
+            // pins the content assertions to the catch-up: without it (or with a
+            // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1
+            // and the per-id checks would fail.
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1);
+
+            // Phase 2: fresh server that reconstructs its per-connection dictionary
+            // from the delta sections. Sender 2 recovers the slot and replays.
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                try (Sender ignored = Sender.fromConfig(cfg)) {
+                    long deadline = System.currentTimeMillis() + 5_000;
+                    while (System.currentTimeMillis() < deadline
+                            && handler.maxDictSize() < DISTINCT_SYMBOLS) {
+                        Thread.sleep(20);
+                    }
+                }
+
+                // The recovering sender must have re-registered the dictionary via a
+                // catch-up (0-table) frame before replaying delta frames.
+                Assert.assertTrue("recovery sent a full-dictionary catch-up frame",
+                        handler.sawCatchUpFrame);
+                // The reconstructed dictionary must be complete and gap-free: exactly
+                // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id.
+                List dict = handler.dictSnapshot();
+                Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size());
+                for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+                    Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i));
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testTornDictionaryRebuildsFromFramesAcrossTheAckWatermark() throws Exception {
+        // A host crash truncates the dictionary to nothing, and the ack watermark sits
+        // mid-stream so the replay set starts at a frame whose delta begins at id 3.
+        //
+        // The old guard called that unreplayable and told the user to resend. It is not: ids
+        // 0..2 are still sitting in the ACKED frames on disk. Trim unlinks whole SEALED
+        // segments, not individual acked frames, so those six tiny frames are all still in
+        // the one active segment. Rebuild from them and the slot drains in full.
+        assertMemoryLeak(() -> {
+            recordSixDeltaFrames();
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            java.nio.file.Path dict = slot.resolve(".symbol-dict");
+            java.nio.file.Files.write(dict,
+                    Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); // header only
+            writeAckWatermark(slot.resolve(".ack-watermark"), 2);
+            assertSlotRecoversWithCompleteDictionary();
+        });
+    }
+
+    @Test
+    public void testUnopenableDictRebuildsFromFramesFromIdZero() throws Exception {
+        // Same rebuild, but the dictionary cannot be OPENED at all (fd exhaustion, a
+        // read-only remount, ENOSPC -- modelled by planting a directory in its place). The
+        // producer's seed used to be gated on the dictionary having opened, which made the
+        // whole rebuild dead code for exactly this case.
+        assertMemoryLeak(() -> {
+            recordSixDeltaFrames();
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            java.nio.file.Path dict = slot.resolve(".symbol-dict");
+            java.nio.file.Files.delete(dict);
+            java.nio.file.Files.createDirectory(dict);
+            writeAckWatermark(slot.resolve(".ack-watermark"), 0);
+            assertSlotRecoversWithCompleteDictionary();
+        });
+    }
+
+    @Test
+    public void testUnopenableDictRebuildsFromFramesAcrossTheAckWatermark() throws Exception {
+        // The hardest of the three: the dictionary is unopenable AND the replay set starts at
+        // deltaStart=3, so ids 0..2 exist nowhere except the acked frames on disk.
+        assertMemoryLeak(() -> {
+            recordSixDeltaFrames();
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            java.nio.file.Path dict = slot.resolve(".symbol-dict");
+            java.nio.file.Files.delete(dict);
+            java.nio.file.Files.createDirectory(dict);
+            writeAckWatermark(slot.resolve(".ack-watermark"), 2);
+            assertSlotRecoversWithCompleteDictionary();
+        });
+    }
+
+    @Test
+    public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() throws Exception {
+        // The genuinely unrecoverable slot -- and the only one left, now that a torn dictionary
+        // rebuilds from the frames that are still on disk.
+        //
+        // Here the frames that REGISTERED the early ids are gone for good: trim unlinked their
+        // whole segment once they were acked (modelled by deleting sf-initial.sfa, which is
+        // exactly what SegmentManager does). The dictionary is torn away too, so nothing
+        // anywhere still holds those ids, and the surviving frames' deltas start above them.
+        // Replaying would null-pad the hole and silently misattribute symbol values.
+        //
+        // So the slot is unreplayable -- and it is SET ASIDE, not allowed to brick the sender.
+        // Failing here is what bricked it: senderId is stable and a not-fully-drained slot is
+        // retained on close, so every retry re-recovered the same slot and threw again, and the
+        // application could not construct a Sender at all -- it could not even buffer new rows.
+        assertMemoryLeak(() -> {
+            writeAndTearUnreplayableSlot();
+
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                // build() must SUCCEED -- on a fresh slot -- and the producer must keep working.
+                try (Sender s2 = Sender.fromConfig(cfg)) {
+                    s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow();
+                    s2.flush();
+                    long deadline = System.currentTimeMillis() + 10_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) {
+                        Thread.sleep(20);
+                    }
+                }
+                assertUnreplayableSlotSetAside();
+                // The producer's new data reaches the server, and not one symbol of the
+                // unreplayable slot does.
+                Assert.assertEquals("the producer must keep producing on its fresh slot",
+                        Arrays.asList("after-recovery"), handler.dictSnapshot());
+            }
+        });
+    }
+
+    @Test
+    public void testQueuedOrphanCannotAdoptSlotWhileQuarantineRecreatesItsName() throws Exception {
+        // Sender 1 leaves a genuinely unreplayable default slot. Sender 2 recovers it,
+        // closes the directory-local lock, quarantines the directory and recreates the
+        // default pathname. A scanner may already have queued that pathname before sender
+        // 2 starts; the queued drainer must not acquire the old lock inode in the close ->
+        // rename gap and later issue path-based writes/unlinks against sender 2's fresh slot.
+        assertMemoryLeak(() -> {
+            writeAndTearUnreplayableSlot();
+            ObjList scannerSnapshot = OrphanScanner.scan(sfDir, "other-sender");
+            Assert.assertEquals(1, scannerSnapshot.size());
+            String staleSnapshotPath = scannerSnapshot.get(0);
+
+            CountDownLatch producerInRenameGap = new CountDownLatch(1);
+            CountDownLatch allowQuarantineRename = new CountDownLatch(1);
+            AtomicReference recoveredSender = new AtomicReference<>();
+            AtomicReference recoveryFailure = new AtomicReference<>();
+            Thread recoveryThread = null;
+            Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest(() -> {
+                producerInRenameGap.countDown();
+                try {
+                    if (!allowQuarantineRename.await(15, TimeUnit.SECONDS)) {
+                        throw new AssertionError("timed out waiting to finish quarantine rename");
+                    }
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    throw new AssertionError("quarantine hook interrupted", e);
+                }
+            });
+            try {
+                DictReconstructingHandler handler = new DictReconstructingHandler();
+                try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                    int port = good.getPort();
+                    good.start();
+                    Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                    String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+
+                    recoveryThread = new Thread(() -> {
+                        try {
+                            recoveredSender.set(Sender.fromConfig(cfg));
+                        } catch (Throwable t) {
+                            recoveryFailure.set(t);
+                        }
+                    }, "qwp-quarantine-recovery");
+                    recoveryThread.start();
+                    Assert.assertTrue("recovering sender did not reach the quarantine gap",
+                            producerInRenameGap.await(10, TimeUnit.SECONDS));
+
+                    BackgroundDrainer queuedDrainer = new BackgroundDrainer(
+                            staleSnapshotPath, 256, 8192, () -> null,
+                            1000, 1, 10, true, 0);
+                    Thread drainerThread = new Thread(queuedDrainer, "qwp-queued-orphan");
+                    drainerThread.start();
+                    drainerThread.join(5_000);
+                    Assert.assertFalse("queued drainer must not wait on or adopt the old inode",
+                            drainerThread.isAlive());
+                    Assert.assertEquals(BackgroundDrainer.DrainOutcome.LOCKED_BY_OTHER,
+                            queuedDrainer.outcome());
+
+                    allowQuarantineRename.countDown();
+                    recoveryThread.join(10_000);
+                    Assert.assertFalse("recovering sender did not finish", recoveryThread.isAlive());
+                    if (recoveryFailure.get() != null) {
+                        throw new AssertionError("recovering sender failed", recoveryFailure.get());
+                    }
+                    Sender sender = recoveredSender.get();
+                    Assert.assertNotNull(sender);
+                    sender.table("m").symbol("s", "after-race").longColumn("v", 99).atNow();
+                    sender.flush();
+                    long deadline = System.currentTimeMillis() + 10_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) {
+                        Thread.sleep(20);
+                    }
+                    sender.close();
+                    recoveredSender.set(null);
+
+                    assertUnreplayableSlotSetAside();
+                    Assert.assertEquals("fresh producer slot must remain intact and usable",
+                            Arrays.asList("after-race"), handler.dictSnapshot());
+                }
+            } finally {
+                allowQuarantineRename.countDown();
+                if (recoveryThread != null) {
+                    recoveryThread.join(10_000);
+                    if (recoveryThread.isAlive()) {
+                        recoveryThread.interrupt();
+                    }
+                }
+                Sender sender = recoveredSender.getAndSet(null);
+                if (sender != null) {
+                    sender.close();
+                }
+                Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest(null);
+            }
+        });
+    }
+
+    @Test
+    public void testFullyAckedTornSlotResumesInPlaceWithoutQuarantine() throws Exception {
+        // M1 regression -- the ACKED counterpart to
+        // testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside. The on-disk
+        // tear is IDENTICAL (earliest segment trimmed, .symbol-dict torn to its header,
+        // so the surviving frames' deltas start above ids nothing on disk still holds),
+        // but here every committed frame was already ACKED before the crash. Nothing is
+        // left to replay, so the "gap" is entirely in data the server already has.
+        //
+        // seedGlobalDictionaryFromPersisted must therefore NOT raise
+        // UnreplayableSlotException. Quarantining a fully-delivered slot would fire a
+        // false "resend required" alarm AND -- because such a slot is fully drained --
+        // let build()'s connect-path close unlink the (already-delivered) bytes the
+        // quarantine claims to preserve. The slot must resume IN PLACE.
+        //
+        // Before the fix the gap detector ignored ack state and threw, so this slot was
+        // set aside as default.unreplayable-0 with a "resend required" error for data the
+        // server had already acknowledged.
+        assertMemoryLeak(() -> {
+            writeAndTearUnreplayableSlot();
+            // Mark every committed frame acked. writeAndTearUnreplayableSlot writes 12
+            // frames (FSNs 0..11), so stamping the watermark at 11 makes
+            // ackedFsn == recoveredCommitBoundaryFsn: a torn dictionary with nothing left
+            // to replay. (Not higher than 11 -- that would make the resuming producer's
+            // next frame look pre-acked.)
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            writeAckWatermark(slot.resolve(".ack-watermark"), 11);
+
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                // build() must succeed WITHOUT setting the slot aside, and the producer
+                // must keep working on the SAME slot.
+                try (Sender s2 = Sender.fromConfig(cfg)) {
+                    s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow();
+                    s2.flush();
+                    long deadline = System.currentTimeMillis() + 10_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) {
+                        Thread.sleep(20);
+                    }
+                    Assert.assertEquals(
+                            "the resumed sender must deliver new data from the original slot",
+                            Arrays.asList("after-recovery"),
+                            handler.dictSnapshot());
+                }
+            }
+            // The fully-acked slot was NOT quarantined: no set-aside copy exists (the
+            // inverse of assertUnreplayableSlotSetAside), and the sender resumed on the
+            // original slot.
+            Assert.assertFalse("a fully-acked torn slot must NOT be quarantined -- its data "
+                            + "was already delivered, so there is nothing to resend",
+                    java.nio.file.Files.isDirectory(Paths.get(sfDir, "default.unreplayable-0")));
+            Assert.assertTrue("the sender must resume on the original slot",
+                    java.nio.file.Files.isDirectory(slot));
+        });
+    }
+
+    private static int countSegmentFiles(java.nio.file.Path dir) {
+        java.io.File[] files = dir.toFile().listFiles();
+        int n = 0;
+        if (files != null) {
+            for (java.io.File f : files) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")) {
+                    n++;
+                }
+            }
+        }
+        return n;
+    }
+
+    @Test
+    public void testQuarantineFailsLoudlyWhenAllSlotNamesSaturated() throws Exception {
+        // M2 regression: when a recovered slot is genuinely unreplayable, build() sets
+        // it aside (quarantineTornSlot) and starts the producer on a fresh slot. But if
+        // it cannot free the slot name -- here every default.unreplayable- candidate
+        // up to MAX_QUARANTINE_SLOT_ATTEMPTS (64) already exists -- it MUST fail LOUDLY:
+        // throw and leave the slot's bytes on disk for a manual resend, never silently
+        // drop data. Only the happy rename path was covered before.
+        assertMemoryLeak(() -> {
+            writeAndTearUnreplayableSlot();
+            // Saturate every quarantine candidate so quarantineTornSlot's rename loop
+            // finds no free name.
+            for (int i = 0; i < 64; i++) {
+                java.nio.file.Files.createDirectories(Paths.get(sfDir, "default.unreplayable-" + i));
+            }
+
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                Sender s = null;
+                try {
+                    s = Sender.fromConfig(cfg);
+                    Assert.fail("build() must throw when the unreplayable slot cannot be set aside");
+                } catch (LineSenderException expected) {
+                    Assert.assertTrue("unexpected message: " + expected.getMessage(),
+                            expected.getMessage().contains("too many quarantined slots already under")
+                                    && expected.getMessage().contains("moved or removed by hand"));
+                } finally {
+                    if (s != null) {
+                        s.close();
+                    }
+                }
+            }
+            // The unreplayable slot's bytes must survive on disk for a manual resend --
+            // the guard fails loudly rather than dropping data.
+            Assert.assertTrue("the slot dir must be preserved", java.nio.file.Files.exists(slot));
+            Assert.assertTrue("the slot's segment data must be preserved",
+                    countSegmentFiles(slot) >= 1);
+        });
+    }
+
+    // Writes 12 delta frames (each introducing a new symbol) into the default slot
+    // across several small segments, then makes the slot GENUINELY unreplayable: trims
+    // the segment holding the earliest ids (munmap + unlink, exactly what SegmentManager
+    // does once they are acked) and tears the .symbol-dict down to its header, so the
+    // surviving frames' deltas start above ids nothing on disk still holds. Recovering
+    // such a slot throws UnreplayableSlotException.
+    private void writeAndTearUnreplayableSlot() throws Exception {
+        try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+            int port = silent.getPort();
+            silent.start();
+            Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+            // Small segments so the frames roll into several files and the earliest ones
+            // can be trimmed away independently.
+            String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                    + ";sf_max_bytes=256;close_flush_timeout_millis=0;";
+            try (Sender s1 = Sender.fromConfig(cfg)) {
+                for (int i = 0; i < 12; i++) {
+                    s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                    s1.flush();
+                }
+            }
+        }
+        java.nio.file.Path slot = Paths.get(sfDir, "default");
+        Assert.assertTrue("the frames must have rolled into more than one segment",
+                countSegmentFiles(slot) > 1);
+        // TRIM the segment that holds the earliest frames.
+        java.nio.file.Files.delete(slot.resolve("sf-initial.sfa"));
+        // ...and tear the dictionary away as well, so nothing holds those ids at all.
+        java.nio.file.Path dict = slot.resolve(".symbol-dict");
+        java.nio.file.Files.write(dict, Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8));
+    }
+
+    @Test
+    public void testUnopenableDictSeedsTheProducerAboveTheRecoveredIds() throws Exception {
+        // The producer must resume ABOVE the ids the recovered frames already define, even when
+        // the dictionary could not be opened.
+        //
+        // seedGlobalDictionaryFromPersisted used to be gated on deltaDictEnabled, which is false
+        // exactly here -- so the producer restarted its id space at 0, on top of ids the
+        // surviving frames define. The send loop's mirror (rebuilt from those frames) still read
+        // id 0 as sym-0 while the producer meant something else by it: the two disagree about
+        // what an id MEANS, which is the whole failure mode the dictionary machinery exists to
+        // prevent.
+        //
+        // Visible on the wire: the new symbol must land ABOVE the recovered ones, not on top of
+        // sym-0.
+        assertMemoryLeak(() -> {
+            recordSixDeltaFrames();
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            java.nio.file.Path dict = slot.resolve(".symbol-dict");
+            java.nio.file.Files.delete(dict);
+            java.nio.file.Files.createDirectory(dict);
+            writeAckWatermark(slot.resolve(".ack-watermark"), 2);
+
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                try (Sender s2 = Sender.fromConfig(cfg)) {
+                    long deadline = System.currentTimeMillis() + 10_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 6) {
+                        Thread.sleep(20);
+                    }
+                    // ...and now the resumed producer introduces a symbol of its own.
+                    s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow();
+                    s2.flush();
+                    deadline = System.currentTimeMillis() + 10_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 7) {
+                        Thread.sleep(20);
+                    }
+                }
+                Assert.assertEquals(
+                        "the resumed producer must take the NEXT id, not reuse id 0 -- reusing it "
+                                + "puts two symbols on one id and silently misattributes values",
+                        Arrays.asList("sym-0", "sym-1", "sym-2", "sym-3", "sym-4", "sym-5",
+                                "after-recovery"),
+                        handler.dictSnapshot());
+            }
+        });
+    }
+
+    @Test
+    public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception {
+        // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's
+        // dictionary and delta baseline from the persisted .symbol-dict, so a
+        // recovered sender that continues ingesting assigns the NEXT id, not a
+        // colliding low one. Without it the new symbol reuses a recovered id and the
+        // fresh server sees a redefinition -> silent misattribution. No prior test
+        // ingests on the recovered sender. Replay is from FSN 0 (no acks), so the
+        // recovered frames legitimately overlap the seeded dictionary -- this also
+        // pins that the redefinition guard does not false-positive on normal
+        // recovery.
+        assertMemoryLeak(() -> {
+            // Phase 1: ingest DISTINCT_SYMBOLS symbols, silent server, close-fast ->
+            // unacked frames + a full persisted dictionary.
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                        + ";sf_max_bytes=4096;close_flush_timeout_millis=0;";
+                try (Sender s1 = Sender.fromConfig(cfg)) {
+                    for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+                        s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                        s1.flush();
+                    }
+                }
+            }
+
+            // Phase 2: recover against a fresh server, then ingest a genuinely NEW
+            // symbol. The producer must continue at id DISTINCT_SYMBOLS.
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                try (Sender s2 = Sender.fromConfig(cfg)) {
+                    s2.table("m").symbol("s", "brand-new").longColumn("v", 99L).atNow();
+                    s2.flush();
+                    long deadline = System.currentTimeMillis() + 10_000;
+                    while (System.currentTimeMillis() < deadline
+                            && handler.maxDictSize() < DISTINCT_SYMBOLS + 1) {
+                        Thread.sleep(20);
+                    }
+                }
+                List dict = handler.dictSnapshot();
+                Assert.assertEquals("recovered sender must continue the dictionary, not collide: " + dict,
+                        DISTINCT_SYMBOLS + 1, dict.size());
+                for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+                    Assert.assertEquals("sym-" + i, dict.get(i));
+                }
+                Assert.assertEquals("brand-new", dict.get(DISTINCT_SYMBOLS));
+            }
+        });
+    }
+
+    @Test
+    public void testUnopenablePersistedDictReplaysSelfSufficientFrameSequence() throws Exception {
+        // Untrimmed counterpart of
+        // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames.
+        //
+        // Same failure mode -- the persisted dictionary cannot be OPENED, so
+        // CursorSendEngine.isDeltaDictEnabled() is false -- but NOTHING was acked, so
+        // replay starts at the FIRST frame. Frames 0..5 carry deltaStart 0..5 and are
+        // COLLECTIVELY self-sufficient: replayed in order from id 0, the server
+        // accumulates the dictionary contiguously, needing no dictionary file at all.
+        //
+        // Pre-fix, accumulateSentDict was gated on deltaDictEnabled while the
+        // torn-dict guard was NOT, so sentDictCount froze at 0: frame 0 (deltaStart=0)
+        // passed the guard, but frame 1 (deltaStart=1 > 0) tripped it and latched a
+        // terminal -- and for the background drainer that means markFailed + a .failed
+        // sentinel, permanently quarantining a slot that drains perfectly. A store-
+        // and-forward contract violation: a TRANSIENT disk condition (fd exhaustion, a
+        // read-only remount, ENOSPC) stranding recoverable data.
+        //
+        // The trimmed sibling pins the other half of the contract: when replay starts
+        // ABOVE the frames that introduced the ids, the terminal is still correct.
+        assertMemoryLeak(() -> {
+            // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i.
+            // The server never acks, so nothing is trimmed and replay starts at frame 0
+            // (the common orphan-drain profile: the server was down the whole time).
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                        + ";close_flush_timeout_millis=0;";
+                try (Sender s1 = Sender.fromConfig(cfg)) {
+                    for (int i = 0; i < 6; i++) {
+                        s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                        s1.flush();
+                    }
+                }
+            }
+
+            // Make the persisted dictionary UNOPENABLE: a directory of the same name
+            // defeats both openRW and openCleanRW, so PersistedSymbolDict.open()
+            // returns null. Deliberately do NOT stamp the ack watermark -- replay must
+            // start at frame 0, which is what makes the sequence self-sufficient.
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            java.nio.file.Path dict = slot.resolve(".symbol-dict");
+            java.nio.file.Files.delete(dict);
+            java.nio.file.Files.createDirectory(dict);
+
+            // Phase 2: recover against a fresh server that reconstructs its per-
+            // connection dictionary from the wire exactly as the real one does --
+            // null-padding any gap, so a gap surfaces as a null rather than passing
+            // silently.
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                LineSenderException terminal = null;
+                Sender s2 = Sender.fromConfig(cfg);
+                try {
+                    long deadline = System.currentTimeMillis() + 10_000;
+                    while (System.currentTimeMillis() < deadline
+                            && handler.maxDictSize() < 6
+                            && terminal == null) {
+                        try {
+                            s2.flush();
+                            Thread.sleep(20);
+                        } catch (LineSenderException e) {
+                            terminal = e;
+                        }
+                    }
+                } finally {
+                    try {
+                        s2.close();
+                    } catch (LineSenderException e) {
+                        if (terminal == null) {
+                            terminal = e;
+                        }
+                    }
+                }
+
+                // Pre-fix: the "symbol dictionary is incomplete" terminal fires on frame 1.
+                Assert.assertNull("a self-sufficient frame sequence must replay without a terminal: "
+                                + (terminal == null ? "" : terminal.getMessage()),
+                        terminal);
+                // Pre-fix: the server dictionary stops at ["sym-0"] -- frame 1 never shipped.
+                List serverDict = handler.dictSnapshot();
+                Assert.assertEquals("every delta frame must replay [dict=" + serverDict + ']',
+                        6, serverDict.size());
+                for (int i = 0; i < 6; i++) {
+                    Assert.assertEquals("id " + i + " must resolve; a null here is a server-side "
+                                    + "null-pad, i.e. a silently NULL symbol column [dict=" + serverDict + ']',
+                            "sym-" + i, serverDict.get(i));
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Exception {
+        // C3 regression: sendCommitMessage does NOT write-ahead-persist the
+        // dictionary, so its frame must carry NO new symbol. A symbol left in the
+        // batch by a cancelled row -- cancelRow rolls back neither
+        // currentBatchMaxSymbolId nor the global-dictionary registration -- must not
+        // ride out on the commit frame: doing so puts an id on the wire that a
+        // recovered slot cannot rebuild from .symbol-dict, diverging the producer
+        // dictionary from the surviving frames and silently misattributing reused
+        // ids after a crash. The commit's delta must be bounded by sentMaxSymbolId
+        // (empty here), not currentBatchMaxSymbolId. Memory mode suffices to observe
+        // the wire behaviour; close() drains every frame to the server first.
+        assertMemoryLeak(() -> {
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                // Transactional + autoFlushRows=1: every completed row auto-flushes
+                // as a DEFERRED batch (setting hasDeferredMessages); an explicit
+                // flush() then emits the commit message.
+                Sender sender = Sender.builder("ws::addr=localhost:" + port + ";")
+                        .transactional(true)
+                        .autoFlushRows(1)
+                        .build();
+                try {
+                    // Row 1 registers "a"@0 and auto-flushes it deferred.
+                    sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow();
+                    // Register "b"@1 on a row that is then cancelled: "b" stays in
+                    // the global dictionary and currentBatchMaxSymbolId advances to
+                    // 1, but nothing persists or sends it.
+                    sender.table("m").symbol("s", "b");
+                    sender.cancelRow();
+                    // Commit the deferred batch. The commit frame must carry an
+                    // EMPTY delta -- NOT "b"@1.
+                    sender.flush();
+                } finally {
+                    sender.close(); // drains every frame (incl. the commit) to the server
+                }
+
+                // The server's reconstructed dictionary must hold ONLY "a". Pre-fix
+                // the commit shipped "b"@1, so the server saw a second symbol.
+                List dict = handler.dictSnapshot();
+                Assert.assertEquals("commit frame must not ship the cancelled row's leaked symbol "
+                                + "(recovery would then diverge from the persisted dictionary): " + dict,
+                        1, dict.size());
+                Assert.assertEquals("a", dict.get(0));
+            }
+        });
+    }
+
+    @Test
+    public void testPersistFailureSurfacesAsLineSenderException() throws Exception {
+        // A dictionary write that fails mid-flush -- a full disk, an exhausted quota --
+        // must reach the caller as a LineSenderException, like every other flush-path
+        // failure. PersistedSymbolDict throws a low-level IllegalStateException on a short
+        // write; persistNewSymbolsBeforePublish wraps it. Without the wrap the raw
+        // IllegalStateException sails straight past every user's
+        // `catch (LineSenderException)` around flush() and takes the application down.
+        //
+        // Nothing could reach that translation before: PersistedSymbolDict has
+        // facade-aware overloads, but CursorSendEngine called only the
+        // FilesFacade.INSTANCE ones, so no test could inject a dictionary I/O fault
+        // through the real producer path. The engine now takes a FilesFacade for exactly
+        // this, and the sender is built on that engine directly -- the same
+        // QwpWebSocketSender.connect(..., CursorSendEngine) entry point Sender.build uses.
+        assertMemoryLeak(() -> {
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+
+                String slot = Paths.get(sfDir, "default").toString();
+                Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir,
+                        io.questdb.client.std.Files.DIR_MODE_DEFAULT));
+                ShortDictWriteFacade ff = new ShortDictWriteFacade();
+                // The engine owns the dictionary; the fault facade reaches only it, so the
+                // segment files still write normally and the ONLY failure is the persist.
+                CursorSendEngine engine = new CursorSendEngine(
+                        slot, 4L * 1024 * 1024, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS,
+                        CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff);
+                Sender sender = QwpWebSocketSender.connect(
+                        "localhost", port, null, 0, 0, 0L, null, false, engine);
+                try {
+                    ff.armed = true; // the next dictionary append short-writes
+                    sender.table("m").symbol("s", "boom").longColumn("v", 1L).atNow();
+                    try {
+                        sender.flush();
+                        Assert.fail("a short write to the symbol dictionary must fail the flush");
+                    } catch (LineSenderException expected) {
+                        Assert.assertTrue("the persist failure must be reported as a sender error, "
+                                        + "not leak a raw IllegalStateException: " + expected.getMessage(),
+                                expected.getMessage().contains("failed to persist symbol dictionary before publish"));
+                    }
+                } finally {
+                    try {
+                        sender.close();
+                    } catch (LineSenderException ignored) {
+                        // close() re-flushes the still-buffered row; the facade has disarmed, so
+                        // this normally succeeds. Either way it is not what we assert.
+                    }
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testReconstructingFixtureResetsAckSequencePerConnection() throws Exception {
+        assertMemoryLeak(() -> {
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                for (int i = 0; i < 2; i++) {
+                    try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+                        sender.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                        long fsn = sender.flushAndGetSequence();
+                        Assert.assertTrue("connection " + (i + 1) + " must receive ACK zero",
+                                sender.awaitAckedFsn(fsn, 5_000));
+                    }
+                }
+
+                Assert.assertEquals("each fresh connection must restart wire ACK sequencing",
+                        Arrays.asList(0L, 0L), handler.ackSequenceStarts());
+            }
+        });
+    }
+
+    @Test
+    public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception {
+        // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs
+        // BEFORE the frame is published (sealAndSwapBuffer -> appendBlocking). If
+        // publish fails (here PAYLOAD_TOO_LARGE, a frame bigger than the SF
+        // segment; a backpressure deadline in production), the frame's symbols are
+        // already on disk but sentMaxSymbolId is NOT advanced and the rows stay
+        // buffered -- so a retry re-runs the persist. Keying the persist range off
+        // pd.size() (not sentMaxSymbolId+1) makes it idempotent. Before that fix,
+        // the retry appended the symbol a SECOND time, breaking the dense
+        // id->position invariant; on recovery every later global id shifts by one
+        // and symbol column values are silently misattributed.
+        assertMemoryLeak(() -> {
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+
+                // Small segment; a heavily padded row's frame cannot fit, so
+                // appendBlocking throws PAYLOAD_TOO_LARGE deterministically -- no
+                // backpressure timing needed. The server never acks (SilentHandler).
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;";
+                String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment
+                Sender sender = Sender.fromConfig(cfg);
+                try {
+                    // Buffer ONE new-symbol row, then flush it TWICE. Each flush
+                    // runs the write-ahead persist and then fails to publish; the
+                    // failed flush leaves the row buffered, so the second flush is
+                    // the retry that (pre-fix) duplicated the persisted symbol.
+                    sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow();
+                    for (int attempt = 0; attempt < 2; attempt++) {
+                        try {
+                            sender.flush();
+                            Assert.fail("oversized frame must fail to publish");
+                        } catch (LineSenderException expected) {
+                            // frame too large -- expected on every attempt
+                        }
+                    }
+
+                    // The persisted dictionary must hold "s0" EXACTLY ONCE.
+                    // Pre-fix, the retry duplicated it (size == 2).
+                    ObjList persisted = ((QwpWebSocketSender) sender).getPersistedSymbolsForTest();
+                    Assert.assertNotNull(persisted);
+                    Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol",
+                            1, persisted.size());
+                    Assert.assertEquals("s0", persisted.getQuick(0));
+                } finally {
+                    try {
+                        sender.close();
+                    } catch (LineSenderException ignored) {
+                        // close() re-flushes the still-buffered oversized row and
+                        // fails again (PAYLOAD_TOO_LARGE); expected here and not
+                        // what we assert. close() still runs its resource cleanup,
+                        // so no native memory leaks.
+                    }
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating() throws Exception {
+        // Regression for the re-encode (else) branch of persistNewSymbolsBeforePublish.
+        // After a failed publish leaves the durable dictionary size ahead of the wire
+        // baseline (pd.size() > sentMaxSymbolId+1), a later flush that introduces a NEW
+        // symbol cannot reuse the frame's already-encoded fast-path bytes -- it
+        // re-encodes just the [pd.size() .. currentBatchMaxSymbolId] suffix via
+        // appendSymbols. Keying that off pd.size() (not sentMaxSymbolId+1) keeps it
+        // idempotent: the already-persisted prefix is NOT re-appended.
+        assertMemoryLeak(() -> {
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                // Small segment: any frame carrying the padded row fails to publish with
+                // PAYLOAD_TOO_LARGE deterministically (no backpressure timing).
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;";
+                String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment
+                Sender sender = Sender.fromConfig(cfg);
+                try {
+                    // Flush 1: s0 is persisted (write-ahead) before the oversized frame
+                    // fails to publish. pd.size()=1, sentMaxSymbolId stays -1.
+                    sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow();
+                    try {
+                        sender.flush();
+                        Assert.fail("oversized frame must fail to publish");
+                    } catch (LineSenderException expected) {
+                    }
+
+                    // Add a NEW symbol s1 (id 1). The failed s0 row is still buffered, so
+                    // the batch is {s0, s1} and the durable size (1) has run ahead of the
+                    // wire baseline (-1) -- the state that selects the appendSymbols branch.
+                    sender.table("m").symbol("s", "s1").stringColumn("p", pad).longColumn("v", 2L).atNow();
+                    try {
+                        sender.flush();
+                        Assert.fail("oversized frame must fail to publish");
+                    } catch (LineSenderException expected) {
+                    }
+
+                    // The else branch persisted ONLY s1 (the suffix). The dictionary holds
+                    // s0, s1 exactly once each. Pre-fix (appendSymbols from
+                    // sentMaxSymbolId+1) re-appended s0, giving size 3.
+                    ObjList persisted = ((QwpWebSocketSender) sender).getPersistedSymbolsForTest();
+                    Assert.assertNotNull(persisted);
+                    Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix",
+                            2, persisted.size());
+                    Assert.assertEquals("s0", persisted.getQuick(0));
+                    Assert.assertEquals("s1", persisted.getQuick(1));
+                } finally {
+                    try {
+                        sender.close();
+                    } catch (LineSenderException ignored) {
+                        // close() re-flushes the still-buffered oversized rows and fails
+                        // again (PAYLOAD_TOO_LARGE); expected, resources still released.
+                    }
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception {
+        // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay.
+        // A failed publish persists the frame's symbol (write-ahead) but does NOT
+        // record the frame, so the persisted dictionary becomes a strict SUPERSET of
+        // the recorded frames' references. A recovering sender must still replay
+        // gap-free: it re-registers the whole (superset) dictionary via the catch-up
+        // -- including the symbol whose frame never reached disk -- so the fresh
+        // server reconstructs a complete, gap-free dictionary. The sibling
+        // testFailedPublishDoesNotDuplicatePersistedSymbols proves the dict has no
+        // duplicate after the failed publish; this proves the resulting slot then
+        // recovers and replays end-to-end against a real server.
+        assertMemoryLeak(() -> {
+            // Phase 1: a silent server (no acks) + a small SF segment. Four small
+            // rows register sym-0..sym-3 and their frames are recorded; a fifth,
+            // oversized row registers (persists) sym-4 but its frame is too large for
+            // the segment, so appendBlocking throws and the frame is NOT recorded.
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port
+                        + ";sf_dir=" + sfDir
+                        + ";sf_max_bytes=4096"
+                        + ";close_flush_timeout_millis=0;";
+                Sender s1 = Sender.fromConfig(cfg);
+                try {
+                    for (int i = 0; i < 4; i++) {
+                        s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                        s1.flush();
+                    }
+                    // Oversized frame: sym-4 is persisted (write-ahead) before the
+                    // publish fails, so the dictionary runs one ahead of the frames.
+                    s1.table("m").symbol("s", "sym-4")
+                            .stringColumn("p", TestUtils.repeat("x", 8000))
+                            .longColumn("v", 4).atNow();
+                    try {
+                        s1.flush();
+                        Assert.fail("oversized frame must fail to publish");
+                    } catch (LineSenderException expected) {
+                        // PAYLOAD_TOO_LARGE -- frame not recorded, sym-4 stays persisted
+                    }
+                } finally {
+                    try {
+                        // Re-flushes the still-buffered oversized row and fails again
+                        // (expected); resources are still released, and the idempotent
+                        // write-ahead does not re-append sym-4.
+                        s1.close();
+                    } catch (LineSenderException ignored) {
+                    }
+                }
+            }
+
+            // The persisted dictionary must hold the superset: sym-0..sym-4 (5 ids),
+            // one more than the four recorded frames reference, with no duplicate.
+            PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString());
+            Assert.assertNotNull(pd);
+            try {
+                Assert.assertEquals("failed publish must leave the dict a superset (sym-0..sym-4)",
+                        5, pd.size());
+            } finally {
+                pd.close();
+            }
+
+            // Phase 2: recover against a fresh server that reconstructs its
+            // dictionary from the wire. The recovering sender must re-register all 5
+            // symbols via a catch-up (sym-4 exists ONLY in the dictionary -- no frame
+            // carries it) and replay the 4 recorded frames, leaving a complete,
+            // gap-free server dictionary.
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                try (Sender ignored = Sender.fromConfig(cfg)) {
+                    long deadline = System.currentTimeMillis() + 5_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 5) {
+                        Thread.sleep(20);
+                    }
+                }
+                Assert.assertTrue("recovery must send a full-dictionary catch-up frame",
+                        handler.sawCatchUpFrame);
+                List dict = handler.dictSnapshot();
+                Assert.assertEquals("recovered dictionary must include the failed-publish symbol",
+                        5, dict.size());
+                for (int i = 0; i < 5; i++) {
+                    Assert.assertEquals("dictionary id " + i + " must be gap-free",
+                            "sym-" + i, dict.get(i));
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Exception {
+        // M2 regression: two DISTINCT source symbols that collapse to the SAME UTF-8
+        // bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get
+        // distinct producer ids and persist as separate entries. On recovery the
+        // producer must rebuild its id space to match the persisted entry count
+        // exactly. Pre-fix, seedGlobalDictionaryFromPersisted used getOrAddSymbol,
+        // which de-duped the two decoded "?" strings, leaving the producer
+        // dictionary (and sentMaxSymbolId) one short of pd.size() -- desyncing from
+        // the send-loop catch-up mirror (which uses pd.size()) and silently
+        // misattributing later symbols after a reconnect. addRecoveredSymbol appends
+        // without de-duping, keeping producer and mirror in lockstep.
+        assertMemoryLeak(() -> {
+            // Phase 1: ingest two lone-surrogate symbols in file mode, close-fast.
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                        + ";close_flush_timeout_millis=0;";
+                try (Sender s1 = Sender.fromConfig(cfg)) {
+                    s1.table("m").symbol("s", "\uD800").longColumn("v", 1L).atNow(); // lone surrogate -> '?'
+                    s1.flush();
+                    s1.table("m").symbol("s", "\uD801").longColumn("v", 2L).atNow(); // a DIFFERENT one -> '?'
+                    s1.flush();
+                }
+            }
+
+            // The persisted dictionary holds TWO entries (both encode to '?').
+            int persistedSize;
+            PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString());
+            Assert.assertNotNull(pd);
+            try {
+                persistedSize = pd.size();
+            } finally {
+                pd.close();
+            }
+            Assert.assertEquals("two colliding symbols must persist as two entries", 2, persistedSize);
+
+            // Phase 2: recover, then introduce a third symbol. The wire must extend
+            // the two recovered ids at id 2 rather than overwrite id 1.
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                try (Sender s2 = Sender.fromConfig(cfg)) {
+                    s2.table("m").symbol("s", "tail").longColumn("v", 3L).atNow();
+                    long fsn = s2.flushAndGetSequence();
+                    Assert.assertTrue("the post-recovery frame must be acknowledged",
+                            s2.awaitAckedFsn(fsn, 5_000));
+                }
+                Assert.assertEquals("the new symbol delta must continue after both recovered entries",
+                        persistedSize, handler.lastDataDeltaStart());
+                Assert.assertEquals("the reconstructed dictionary must retain both colliding entries "
+                                + "and append the new symbol at id 2",
+                        Arrays.asList("?", "?", "tail"), handler.dictSnapshot());
+            }
+        });
+    }
+
+    @Test
+    public void testTornDictSubsetRebuildsFromSurvivingFrames() throws Exception {
+        // The persisted .symbol-dict is NOT fsync'd, so a host crash can lose its
+        // highest-id tail entries while the segment frames that introduced those ids
+        // survive (out-of-order page loss). Those frames carry the torn-off symbols in
+        // their OWN delta sections -- which is exactly why the background drainer replays
+        // such a slot perfectly: accumulateSentDict rebuilds the dictionary from them.
+        //
+        // The producer must do the same. Pre-fix it seeded ONLY from the short dictionary,
+        // saw that the frames out-reached it, and threw -- permanently bricking
+        // Sender.build() for a slot the drainer drains fine. And the brick is permanent:
+        // build()'s catch releases the slot lock, but the sender that would have hosted the
+        // orphan drainer is the one that just failed, so nothing drains the slot and the
+        // retry recovers it and throws again. seedGlobalDictionaryFromPersisted now seeds
+        // from the dictionary's intact prefix AND THEN from the surviving frames' deltas,
+        // so the producer's baseline lands on exactly the coverage the send loop's mirror
+        // will reach.
+        //
+        // testTornDictionaryFailsCleanlyInsteadOfCorrupting pins the other half of the
+        // contract: when the symbol-introducing frames were acked and TRIMMED away, the ids
+        // really are unrecoverable and the resume must still fail clean.
+        assertMemoryLeak(() -> {
+            // Phase 1: three delta frames (a@0, b@1, c@2) with nothing acked, so all three
+            // survive and replay from frame 0.
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                        + ";close_flush_timeout_millis=0;";
+                Sender s1 = Sender.fromConfig(cfg);
+                try {
+                    s1.table("m").symbol("s", "a").longColumn("v", 0).atNow();
+                    s1.flush();
+                    s1.table("m").symbol("s", "b").longColumn("v", 1).atNow();
+                    s1.flush();
+                    s1.table("m").symbol("s", "c").longColumn("v", 2).atNow();
+                    s1.flush();
+                } finally {
+                    s1.close();
+                }
+            }
+
+            // Simulate the host-crash tear: rewrite the dictionary to drop its highest
+            // entry (c@2), keeping a@0,b@1 -- while the frame that introduced c@2 stays on
+            // disk. openClean truncates; the two appends leave exactly the two-entry
+            // dictionary a torn tail would recover to.
+            String slotDir = Paths.get(sfDir, "default").toString();
+            try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) {
+                Assert.assertNotNull(torn);
+                torn.appendSymbol("a");
+                torn.appendSymbol("b");
+                Assert.assertEquals(2, torn.size());
+            }
+
+            // Phase 2: the resuming sender must REBUILD c@2 from that surviving frame,
+            // replay everything, and keep ingesting ABOVE the recovered tip.
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                // Pre-fix this line THREW ("subset of the surviving frames").
+                try (Sender resumed = Sender.fromConfig(cfg)) {
+                    long deadline = System.currentTimeMillis() + 5_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 3) {
+                        Thread.sleep(20);
+                    }
+                    // A NEW symbol must land ABOVE the recovered tip. Seeded short, the
+                    // producer would hand "d" an id the surviving frames already define,
+                    // silently overwriting that symbol on the server -- the corruption the
+                    // old throw existed to prevent, and which this seeding removes at the
+                    // source rather than by refusing to start.
+                    resumed.table("m").symbol("s", "d").longColumn("v", 3).atNow();
+                    resumed.flush();
+                    deadline = System.currentTimeMillis() + 5_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 4) {
+                        Thread.sleep(20);
+                    }
+                }
+
+                List dict = handler.dictSnapshot();
+                Assert.assertEquals("dictionary must rebuild gap-free [" + dict + ']', 4, dict.size());
+                Assert.assertEquals("a", dict.get(0));
+                Assert.assertEquals("b", dict.get(1));
+                // Rebuilt from the surviving frame's own delta, not from the torn file.
+                Assert.assertEquals("c", dict.get(2));
+                // The new symbol, placed above the recovered tip -- no id reuse.
+                Assert.assertEquals("d", dict.get(3));
+            }
+        });
+    }
+
+    @Test
+    public void testTornDictTotalLossRebuildsFromSurvivingFrames() throws Exception {
+        // The total-loss counterpart of testTornDictSubsetRebuildsFromSurvivingFrames: the
+        // whole dictionary is gone (size 0) but the file still opens, so
+        // isDeltaDictEnabled() stays true and the engine does NOT discard it (the frames
+        // carry deltaStart 0,1,2, so maxSymbolDeltaStart != 0 and the full-dict-fallback
+        // discard correctly stays out of the way).
+        //
+        // The surviving frames start at id 0 and are collectively self-sufficient, so the
+        // producer can rebuild the ENTIRE dictionary from their deltas -- exactly as the
+        // send loop's mirror does. Pre-fix this threw and bricked build().
+        assertMemoryLeak(() -> {
+            // Phase 1: a@0, b@1, c@2; nothing acked, so all three replay from frame 0.
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                        + ";close_flush_timeout_millis=0;";
+                Sender s1 = Sender.fromConfig(cfg);
+                try {
+                    s1.table("m").symbol("s", "a").longColumn("v", 0).atNow();
+                    s1.flush();
+                    s1.table("m").symbol("s", "b").longColumn("v", 1).atNow();
+                    s1.flush();
+                    s1.table("m").symbol("s", "c").longColumn("v", 2).atNow();
+                    s1.flush();
+                } finally {
+                    s1.close();
+                }
+            }
+
+            // Total tear: the dictionary recovers EMPTY but still opens.
+            String slotDir = Paths.get(sfDir, "default").toString();
+            try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) {
+                Assert.assertNotNull(torn);
+                Assert.assertEquals(0, torn.size());
+            }
+
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                // Pre-fix this line THREW.
+                try (Sender resumed = Sender.fromConfig(cfg)) {
+                    long deadline = System.currentTimeMillis() + 5_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 3) {
+                        Thread.sleep(20);
+                    }
+                    resumed.table("m").symbol("s", "d").longColumn("v", 3).atNow();
+                    resumed.flush();
+                    deadline = System.currentTimeMillis() + 5_000;
+                    while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 4) {
+                        Thread.sleep(20);
+                    }
+                }
+
+                List dict = handler.dictSnapshot();
+                Assert.assertEquals("the whole dictionary must rebuild from the frames [" + dict + ']',
+                        4, dict.size());
+                Assert.assertEquals("a", dict.get(0));
+                Assert.assertEquals("b", dict.get(1));
+                Assert.assertEquals("c", dict.get(2));
+                Assert.assertEquals("d", dict.get(3));
+            }
+            // The side-file is healed in passing -- persistNewSymbolsBeforePublish resumes
+            // from pd.size() (0 here), so the flush above re-persisted the rebuilt ids
+            // a,b,c along with d. That is not asserted on disk because it is not
+            // observable here: the slot drains fully, and a fully-drained close removes
+            // the dictionary (CursorSendEngine -> PersistedSymbolDict.removeOrphan), so
+            // re-opening it would just yield a fresh empty file. The persist-resumes-from-
+            // pd.size() behaviour is pinned by the testFailedPublish* tests.
+        });
+    }
+
+    @Test
+    public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Exception {
+        // M1 regression (counterpoint to testTornDictTotalLossFailsCleanOnResume): a
+        // slot written in FULL-DICT fallback -- the .symbol-dict could not open when
+        // writing, so isDeltaDictEnabled() was false and every frame re-ships the
+        // whole dictionary from id 0 (deltaStart=0) -- leaves SELF-SUFFICIENT frames
+        // and NO side-file. On recovery the engine opens a FRESH EMPTY .symbol-dict,
+        // so the surviving frames out-reach it (recoveredMaxSymbolId >= pd.size()==0).
+        // Those frames carry their whole dictionary inline and need no side-file, so
+        // they must RECOVER, not brick build(). CursorSendEngine detects them
+        // (maxSymbolDeltaStart == 0) and discards the empty side-file, recovering the
+        // slot in full-dict mode. Before the fix the sender's seed-time guard treated
+        // the empty dictionary as a host-crash tear and threw from Sender.build(),
+        // even though the orphan drainer drains the very same frames fine. A torn
+        // DELTA dictionary (deltaStart > 0) still fails clean -- see that other test.
+        assertMemoryLeak(() -> {
+            java.nio.file.Path slot = Paths.get(sfDir, "default");
+            java.nio.file.Path dict = slot.resolve(".symbol-dict");
+            // Force full-dict fallback in phase 1: plant a non-empty DIRECTORY where
+            // the dictionary file belongs, so openCleanRW fails and delta encoding
+            // stays disabled -- the frames are then written self-sufficient.
+            java.nio.file.Files.createDirectories(dict);
+            java.nio.file.Path blocker = dict.resolve("blocker");
+            java.nio.file.Files.createFile(blocker);
+
+            // Phase 1: silent server (no acks). Sender 1 writes new-symbol rows in
+            // full-dict mode and close-fast, leaving unacked self-sufficient frames.
+            try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+                int port = silent.getPort();
+                silent.start();
+                Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port
+                        + ";sf_dir=" + sfDir
+                        + ";close_flush_timeout_millis=0;";
+                try (Sender s1 = Sender.fromConfig(cfg)) {
+                    for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+                        s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                        s1.flush();
+                    }
+                }
+                // The planted directory is untouched -- the dictionary never opened,
+                // so the frames were written full-dict with no side-file.
+                Assert.assertTrue("full-dict fallback: .symbol-dict must stay a directory",
+                        java.nio.file.Files.isDirectory(dict));
+            }
+
+            // Drop the planted directory so recovery opens a FRESH EMPTY .symbol-dict
+            // where it belongs -- exactly the state a full-dict-fallback slot recovers
+            // into (frames on disk, no dictionary behind them).
+            java.nio.file.Files.delete(blocker);
+            java.nio.file.Files.delete(dict);
+
+            // Phase 2: recover. build() must SUCCEED (not throw the torn-dict guard),
+            // and the self-sufficient frames replay against a fresh server gap-free.
+            DictReconstructingHandler handler = new DictReconstructingHandler();
+            try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+                int port = good.getPort();
+                good.start();
+                Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+                String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                try (Sender ignored = Sender.fromConfig(cfg)) { // must NOT throw
+                    long deadline = System.currentTimeMillis() + 5_000;
+                    while (System.currentTimeMillis() < deadline
+                            && handler.maxDictSize() < DISTINCT_SYMBOLS) {
+                        Thread.sleep(20);
+                    }
+                }
+                // Full-dict recovery re-ships the whole dictionary inline on every
+                // frame, so there is NO 0-table catch-up frame (that is the delta-mode
+                // reconnect path). The reconstructed dictionary is still gap-free.
+                Assert.assertFalse("full-dict recovery must NOT send a catch-up frame",
+                        handler.sawCatchUpFrame);
+                List reconstructed = handler.dictSnapshot();
+                Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, reconstructed.size());
+                for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+                    Assert.assertEquals("dictionary id " + i, "sym-" + i, reconstructed.get(i));
+                }
+            }
+        });
+    }
+
+    private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException {
+        byte[] buf = new byte[16];
+        ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
+        bb.putInt(0x31574B41); // 'AKW1'
+        bb.putInt(0);          // reserved
+        bb.putLong(fsn);
+        java.nio.file.Files.write(path, buf);
+    }
+
+    /**
+     * Reconstructs the per-connection symbol dictionary from delta sections,
+     * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding.
+     */
+    private static class DictReconstructingHandler implements TestWebSocketServer.WebSocketServerHandler {
+        volatile boolean sawCatchUpFrame;
+        private final List ackSequenceStarts = new ArrayList<>();
+        private final List dict = new ArrayList<>();
+        private final AtomicLong nextSeq = new AtomicLong(0);
+        private TestWebSocketServer.ClientHandler currentClient;
+        private volatile int lastDataDeltaStart = -1;
+
+        synchronized List ackSequenceStarts() {
+            return new ArrayList<>(ackSequenceStarts);
+        }
+
+        synchronized List dictSnapshot() {
+            return new ArrayList<>(dict);
+        }
+
+        synchronized int maxDictSize() {
+            return dict.size();
+        }
+
+        int lastDataDeltaStart() {
+            return lastDataDeltaStart;
+        }
+
+        @Override
+        public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            boolean newConnection = currentClient != client;
+            if (newConnection) {
+                currentClient = client;
+                dict.clear(); // fresh server dictionary per connection
+                nextSeq.set(0);
+            }
+            QwpWireTestUtils.accumulateDeltaDictionary(data, dict);
+            int tableCount = QwpWireTestUtils.tableCount(data);
+            if (tableCount == 0 && QwpWireTestUtils.hasDelta(data)) {
+                sawCatchUpFrame = true;
+            } else if (tableCount > 0 && QwpWireTestUtils.hasDelta(data)) {
+                lastDataDeltaStart = QwpWireTestUtils.readVarint(data, new int[]{12});
+            }
+            try {
+                long ackSequence = nextSeq.getAndIncrement();
+                if (newConnection) {
+                    ackSequenceStarts.add(ackSequence);
+                }
+                client.sendBinary(QwpWireTestUtils.buildAck(ackSequence));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler {
+        @Override
+        public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            // never acks -- sender leaves everything unacked in the slot
+        }
+    }
+
+    /** Counts every binary frame it receives and acks it. */
+    private static class CountingHandler implements TestWebSocketServer.WebSocketServerHandler {
+        final AtomicInteger frames = new AtomicInteger();
+        private final AtomicLong nextSeq = new AtomicLong(0);
+
+        @Override
+        public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            frames.incrementAndGet();
+            try {
+                client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+    /**
+     * Short-writes the FIRST dictionary append after it is armed, modelling a disk that
+     * fills mid-flush. Offset 0 is left alone so the file header still writes -- only the
+     * entry append fails, which is the path under test.
+     */
+    private static final class ShortDictWriteFacade extends DelegatingFilesFacade {
+        boolean armed;
+
+        @Override
+        public long write(int fd, long addr, long len, long offset) {
+            if (armed && offset > 0 && len > 1) {
+                armed = false;
+                return INSTANCE.write(fd, addr, len - 1, offset);
+            }
+            return INSTANCE.write(fd, addr, len, offset);
+        }
+    }
+
+
+    /**
+     * The unreplayable-slot contract: a recovered slot whose symbol dictionary cannot be
+     * rebuilt -- not from its own intact prefix, not from the surviving frames' delta
+     * sections -- is SET ASIDE, never silently drained and never allowed to brick the
+     * sender.
+     * 

+     * The bytes are kept for forensics and resend, the {@code .failed} sentinel tells the
+     * orphan drainer to treat the copy as human-in-the-loop rather than retry it forever,
+     * and the producer continues on a fresh, empty slot.
+     */
+    private void assertUnreplayableSlotSetAside() {
+        java.nio.file.Path aside = Paths.get(sfDir, "default.unreplayable-0");
+        Assert.assertTrue("the unreplayable slot must be set aside, not deleted: " + aside,
+                java.nio.file.Files.isDirectory(aside));
+        Assert.assertTrue("the set-aside slot must keep its recorded frames for resend",
+                hasSegmentFile(aside));
+        Assert.assertTrue("the set-aside slot must carry the .failed sentinel",
+                java.nio.file.Files.exists(aside.resolve(".failed")));
+        Assert.assertTrue("the sender must continue on a live slot",
+                java.nio.file.Files.isDirectory(Paths.get(sfDir, "default")));
+    }
+
+    private static boolean hasSegmentFile(java.nio.file.Path dir) {
+        java.io.File[] files = dir.toFile().listFiles();
+        if (files != null) {
+            for (java.io.File f : files) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")
+                        && f.length() > 0) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+
+    /**
+     * Phase 1 for the recovery tests: six frames, each introducing exactly one new symbol
+     * (sym-0 .. sym-5), left unacked on disk by a silent server and a fast close.
+     */
+    private void recordSixDeltaFrames() throws Exception {
+        try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+            int port = silent.getPort();
+            silent.start();
+            Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+            String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                    + ";close_flush_timeout_millis=0;";
+            try (Sender s1 = Sender.fromConfig(cfg)) {
+                for (int i = 0; i < 6; i++) {
+                    s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                    s1.flush();
+                }
+            }
+        }
+    }
+
+    /**
+     * Recovers the slot against a fresh server and asserts the dictionary it ends up holding
+     * is COMPLETE and IN ORDER.
+     * 

+     * This is the strong form, and it is the point: the fresh server starts with an empty
+     * dictionary, so ids the replayed frames' deltas start ABOVE can only come from a catch-up
+     * frame -- which can only carry them if the mirror was seeded from the frames still on
+     * disk. A dictionary that came back null-padded (the server's response to an id it has
+     * never seen) or shifted by one would fail here, and that is precisely the corruption the
+     * old guard condemned these slots to avoid. It never had to.
+     */
+    private void assertSlotRecoversWithCompleteDictionary() throws Exception {
+        DictReconstructingHandler handler = new DictReconstructingHandler();
+        try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+            int port = good.getPort();
+            good.start();
+            Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+            String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+            try (Sender s2 = Sender.fromConfig(cfg)) {
+                long deadline = System.currentTimeMillis() + 10_000;
+                while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 6) {
+                    Thread.sleep(20);
+                }
+                s2.flush();
+            }
+            Assert.assertEquals(
+                    "every id is still held by a frame on disk, so the catch-up must rebuild the "
+                            + "dictionary COMPLETE and gap-free -- not null-pad it",
+                    Arrays.asList("sym-0", "sym-1", "sym-2", "sym-3", "sym-4", "sym-5"),
+                    handler.dictSnapshot());
+        }
+    }
+
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
index b8945d40..2f38ab02 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
@@ -31,6 +31,36 @@
 
 public class GlobalSymbolDictionaryTest {
 
+    @Test
+    public void testAddRecoveredSymbol_appendsWithoutDeduplicating() {
+        // Recovery replays persisted entries in id order. Distinct source strings
+        // that decode to the same characters -- lone UTF-16 surrogates both
+        // UTF-8-encode to '?', so they read back as the string "?" -- must keep
+        // DISTINCT ids, so the producer id space matches the persisted entry count.
+        // getOrAddSymbol de-dups them; addRecoveredSymbol must not.
+        GlobalSymbolDictionary dedup = new GlobalSymbolDictionary();
+        dedup.getOrAddSymbol("?");
+        dedup.getOrAddSymbol("?");
+        assertEquals("getOrAddSymbol de-dups colliding strings", 1, dedup.size());
+
+        GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+        assertEquals(0, recovered.addRecoveredSymbol("?"));
+        assertEquals(1, recovered.addRecoveredSymbol("?"));
+        assertEquals(2, recovered.addRecoveredSymbol("nvda"));
+        assertEquals("addRecoveredSymbol keeps colliding entries distinct", 3, recovered.size());
+
+        // Dense id -> symbol mapping is preserved position-for-position.
+        assertEquals("?", recovered.getSymbol(0));
+        assertEquals("?", recovered.getSymbol(1));
+        assertEquals("nvda", recovered.getSymbol(2));
+
+        // A later ingest of a colliding string reuses the highest recovered id
+        // (harmless -- both encode to identical bytes), and a genuinely new symbol
+        // continues past the recovered tip.
+        assertEquals(1, recovered.getOrAddSymbol("?"));
+        assertEquals(3, recovered.getOrAddSymbol("brand-new"));
+    }
+
     @Test
     public void testAddSymbol_assignsSequentialIds() {
         GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java
new file mode 100644
index 00000000..d2770145
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java
@@ -0,0 +1,96 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.ClientTlsConfiguration;
+import io.questdb.client.Sender;
+import io.questdb.client.SenderConnectionListener;
+import io.questdb.client.SenderErrorHandler;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import org.junit.Test;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class QwpPublicApiCompatibilityTest {
+
+    @Test
+    public void testBackgroundDrainerLegacyConstructorDescriptorIsPreserved() throws Exception {
+        Constructor constructor = BackgroundDrainer.class.getConstructor(
+                String.class,
+                long.class,
+                long.class,
+                CursorWebSocketSendLoop.ReconnectFactory.class,
+                long.class,
+                long.class,
+                long.class,
+                boolean.class,
+                long.class,
+                int.class,
+                long.class
+        );
+        assertTrue(Modifier.isPublic(constructor.getModifiers()));
+    }
+
+    @Test
+    public void testQwpWebSocketSenderLegacyConnectDescriptorIsPreserved() throws Exception {
+        Method method = QwpWebSocketSender.class.getMethod(
+                "connect",
+                List.class,
+                ClientTlsConfiguration.class,
+                int.class,
+                int.class,
+                long.class,
+                String.class,
+                boolean.class,
+                CursorSendEngine.class,
+                long.class,
+                long.class,
+                long.class,
+                long.class,
+                Sender.InitialConnectMode.class,
+                SenderErrorHandler.class,
+                int.class,
+                long.class,
+                long.class,
+                int.class,
+                SenderConnectionListener.class,
+                int.class,
+                int.class,
+                long.class
+        );
+        assertEquals(QwpWebSocketSender.class, method.getReturnType());
+        assertTrue(Modifier.isPublic(method.getModifiers()));
+        assertTrue(Modifier.isStatic(method.getModifiers()));
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java
index 4d7d7931..5acc0cf9 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java
@@ -25,6 +25,7 @@
 package io.questdb.client.test.cutlass.qwp.client;
 
 import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer;
 import io.questdb.client.cutlass.qwp.client.QwpBufferWriter;
 import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
@@ -1284,6 +1285,81 @@ public void testReset() throws Exception {
         });
     }
 
+    @Test
+    public void testSplitMessageCopiesStagedTableBodies() throws Exception {
+        assertMemoryLeak(() -> {
+            try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder();
+                 MicrobatchBuffer target = new MicrobatchBuffer(64);
+                 QwpTableBuffer t0 = new QwpTableBuffer("alpha");
+                 QwpTableBuffer t1 = new QwpTableBuffer("bravo")) {
+                GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+                int aapl = dict.getOrAddSymbol("AAPL"); // 0
+                int goog = dict.getOrAddSymbol("GOOG"); // 1
+
+                t0.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("AAPL", aapl);
+                t0.getOrCreateColumn("v", TYPE_LONG, false).addLong(1L);
+                t0.nextRow();
+
+                t1.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("GOOG", goog);
+                t1.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(2.5);
+                t1.getOrCreateColumn("s", TYPE_VARCHAR, true).addString("hello");
+                t1.nextRow();
+
+                encoder.beginMessage(2, dict, -1, 1);
+                int t0BodyOffset = encoder.getBuffer().getPosition();
+                encoder.addTable(t0);
+                int t0BodyLength = encoder.getBuffer().getPosition() - t0BodyOffset;
+                int t1BodyOffset = encoder.getBuffer().getPosition();
+                encoder.addTable(t1);
+                int t1BodyLength = encoder.getBuffer().getPosition() - t1BodyOffset;
+                encoder.finishMessage();
+                long staged = encoder.getBuffer().getBufferPtr();
+
+                // Erase the source tables after their one combined encode. Split
+                // assembly must still reproduce both bodies from staged bytes.
+                t0.clear();
+                t1.clear();
+
+                int firstSize = encoder.copySplitMessage(target, t0BodyOffset, t0BodyLength,
+                        true, -1, 1);
+                long first = target.getBufferPtr();
+                Assert.assertEquals(firstSize, target.getBufferPos());
+                Assert.assertEquals(1, Unsafe.getUnsafe().getShort(first + 6));
+                Assert.assertEquals(firstSize - HEADER_SIZE, Unsafe.getUnsafe().getInt(first + 8));
+                Assert.assertEquals(FLAG_DEFER_COMMIT,
+                        (byte) (Unsafe.getUnsafe().getByte(first + HEADER_OFFSET_FLAGS) & FLAG_DEFER_COMMIT));
+                Cursor cursor = new Cursor(first + HEADER_SIZE);
+                Assert.assertEquals(0, cursor.readVarint());
+                Assert.assertEquals(2, cursor.readVarint());
+                Assert.assertEquals("AAPL", cursor.readString());
+                Assert.assertEquals("GOOG", cursor.readString());
+                for (int i = 0; i < t0BodyLength; i++) {
+                    Assert.assertEquals("first staged table body byte " + i,
+                            Unsafe.getUnsafe().getByte(staged + t0BodyOffset + i),
+                            Unsafe.getUnsafe().getByte(cursor.address + i));
+                }
+
+                target.reset();
+                int secondSize = encoder.copySplitMessage(target, t1BodyOffset, t1BodyLength,
+                        false, 1, 1);
+                long second = target.getBufferPtr();
+                Assert.assertEquals(secondSize, target.getBufferPos());
+                Assert.assertEquals(1, Unsafe.getUnsafe().getShort(second + 6));
+                Assert.assertEquals(secondSize - HEADER_SIZE, Unsafe.getUnsafe().getInt(second + 8));
+                Assert.assertEquals(0,
+                        Unsafe.getUnsafe().getByte(second + HEADER_OFFSET_FLAGS) & FLAG_DEFER_COMMIT);
+                cursor = new Cursor(second + HEADER_SIZE);
+                Assert.assertEquals(2, cursor.readVarint());
+                Assert.assertEquals(0, cursor.readVarint());
+                for (int i = 0; i < t1BodyLength; i++) {
+                    Assert.assertEquals("second staged table body byte " + i,
+                            Unsafe.getUnsafe().getByte(staged + t1BodyOffset + i),
+                            Unsafe.getUnsafe().getByte(cursor.address + i));
+                }
+            }
+        });
+    }
+
     @Test
     public void testVersionByteInHeader() throws Exception {
         assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
new file mode 100644
index 00000000..2d2cc0b1
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
@@ -0,0 +1,95 @@
+/*+*****************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT;
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
+
+final class QwpWireTestUtils {
+
+    static void accumulateDeltaDictionary(byte[] frame, List dictionary) {
+        if (!hasDelta(frame)) {
+            return;
+        }
+        int[] position = {HEADER_SIZE};
+        int deltaStart = readVarint(frame, position);
+        int deltaCount = readVarint(frame, position);
+        while (dictionary.size() < deltaStart) {
+            dictionary.add(null);
+        }
+        for (int i = 0; i < deltaCount; i++) {
+            int length = readVarint(frame, position);
+            String symbol = new String(frame, position[0], length, StandardCharsets.UTF_8);
+            position[0] += length;
+            int symbolId = deltaStart + i;
+            while (dictionary.size() <= symbolId) {
+                dictionary.add(null);
+            }
+            dictionary.set(symbolId, symbol);
+        }
+    }
+
+    static byte[] buildAck(long sequence) {
+        byte[] buffer = new byte[1 + Long.BYTES + Short.BYTES];
+        ByteBuffer byteBuffer = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN);
+        byteBuffer.put((byte) 0x00);
+        byteBuffer.putLong(sequence);
+        byteBuffer.putShort((short) 0);
+        return buffer;
+    }
+
+    static boolean hasDelta(byte[] frame) {
+        return frame.length >= HEADER_SIZE && (frame[5] & FLAG_DELTA_SYMBOL_DICT) != 0;
+    }
+
+    static int readVarint(byte[] buffer, int[] position) {
+        int result = 0;
+        int shift = 0;
+        while (position[0] < buffer.length) {
+            int b = buffer[position[0]++] & 0xFF;
+            result |= (b & 0x7F) << shift;
+            if ((b & 0x80) == 0) {
+                return result;
+            }
+            shift += 7;
+            if (shift > 28) {
+                throw new IllegalStateException("varint too long");
+            }
+        }
+        throw new IllegalStateException("varint truncated");
+    }
+
+    static int tableCount(byte[] frame) {
+        return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8);
+    }
+
+    private QwpWireTestUtils() {
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java
index 36553388..72c8c021 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java
@@ -25,7 +25,6 @@
 package io.questdb.client.test.cutlass.qwp.client;
 
 import io.questdb.client.Sender;
-import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
 import org.junit.Assert;
 import org.junit.Test;
@@ -52,9 +51,10 @@
  * disconnect = sender broken, every subsequent batch threw. Reconnect
  * machinery now handles transient drops: detect, build a fresh client
  * via the registered factory, reset wire state, and reposition the replay
- * cursor at {@code engine.ackedFsn() + 1}. Cursor frames are self-sufficient
- * (every frame carries full schema + full symbol-dict delta), so post-reconnect
- * replay needs no producer-side schema-reset signal.
+ * cursor at {@code engine.ackedFsn() + 1}. Every frame carries its full schema
+ * inline; the symbol dictionary is either shipped in full per frame (file-mode
+ * store-and-forward) or re-registered by an I/O-thread catch-up frame before
+ * replay (memory-mode), so post-reconnect replay needs no producer-side reset.
  * 

  * This commit covers the mechanics with a single-attempt retry; backoff,
  * per-outage time cap, and auth-failure detection follow.
@@ -202,14 +202,13 @@ public void testReconnectNeverGivesUpInvariantB() throws Exception {
     }
 
     @Test
-    public void testTerminalUpgradeErrorAbortsReconnect() throws Exception {
+    public void testPostStartAuthErrorRemainsBuffered() throws Exception {
         // Bespoke raw-socket fixture: first connection completes the
         // WebSocket upgrade and feeds back STATUS_OK ACKs; any subsequent
         // connection gets HTTP 401 Unauthorized — exercising the
-        // auth-terminal path. With reconnect_max_duration_millis=10s and
-        // a 401 happening on the very first reconnect, the cursor I/O
-        // loop should surface the terminal error within hundreds of ms,
-        // not after 10s.
+        // post-start auth-rotation path. Once a foreground sender has been
+        // live, store-and-forward owns its unacked data: repeated 401s must
+        // remain contained and retried rather than stopping the producer.
         try (Auth401AfterFirstConnectionFixture fixture =
                      new Auth401AfterFirstConnectionFixture()) {
             fixture.start();
@@ -223,9 +222,8 @@ public void testTerminalUpgradeErrorAbortsReconnect() throws Exception {
                 // Wait for first connection to ACK + close
                 waitFor(() -> fixture.acceptedConnections.get() >= 2, 5_000);
 
-                long t0 = System.nanoTime();
                 Throwable observed = null;
-                long deadline = System.currentTimeMillis() + 5_000;
+                long deadline = System.currentTimeMillis() + 750;
                 while (System.currentTimeMillis() < deadline) {
                     try {
                         sender.table("foo").longColumn("v", 2L).atNow();
@@ -236,23 +234,10 @@ public void testTerminalUpgradeErrorAbortsReconnect() throws Exception {
                     }
                     Thread.sleep(50);
                 }
-                long elapsedMs = (System.nanoTime() - t0) / 1_000_000L;
-                Assert.assertNotNull("expected terminal error after auth rejection",
+                Assert.assertNull("post-start auth rejection must stay buffered and retriable",
                         observed);
-                Assert.assertTrue(
-                        "terminal upgrade error must surface well inside the cap; took "
-                                + elapsedMs + "ms (cap was 10000ms)",
-                        elapsedMs < 5_000);
-                String msg = observed.getMessage() == null ? "" : observed.getMessage();
-                Assert.assertTrue(
-                        "error must mention the terminal upgrade failure: " + msg,
-                        msg.contains("WebSocket upgrade failed")
-                                || msg.contains("I/O thread failed")
-                                || msg.contains("401"));
-            } catch (LineSenderException ignored) {
+                waitFor(() -> fixture.acceptedConnections.get() >= 3, 5_000);
             }
-            // close() rethrows the latched terminal upgrade error
-            // (commit 052f6ee). Already observed and asserted above.
         }
     }
 
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
index e868dafc..274d39bf 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
@@ -154,7 +154,9 @@ private static int countPopulatedSegmentFiles(String dir) {
             int rc = 1;
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
-                if (name != null && name.endsWith(".sfa")) {
+                if (name != null
+                        && name.endsWith(".sfa")
+                        && !name.startsWith(".qwp-v2-guard-")) {
                     try {
                         try (MmapSegment seg = MmapSegment.openExisting(dir + "/" + name)) {
                             if (seg.frameCount() > 0) n++;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
index 45275528..0c88bc11 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
@@ -25,30 +25,38 @@
 package io.questdb.client.test.cutlass.qwp.client;
 
 import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.std.ObjList;
 import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
 /**
- * Pins down the "every frame on disk is self-sufficient" rule for the symbol
- * dictionary.
+ * Pins down how the symbol dictionary is framed on the wire.
  * 

- * The cursor SF path used to elide previously-sent symbols on subsequent
- * batches over the same connection, emitting a delta-dict that carried only
- * the new entries. That's wrong for SF: the bytes survive process restarts and
- * replay against fresh server connections (post-reconnect, or via a background
- * drainer adopting an orphan slot). A delta that references symbol ids the new
- * server has never seen is unrecoverable.
+ * Both engine modes ship monotonic deltas -- each symbol id travels once,
+ * not the whole dictionary per message -- which is the bandwidth win this feature
+ * adds. The I/O thread re-registers the dictionary with a catch-up frame whenever
+ * it (re)connects, so a fresh server can resolve the non-self-sufficient delta
+ * frames that follow.
  * 

- * Today every frame must carry a complete symbol-dict delta starting at id 0
- * (column schemas travel inline on the first batch too). This test asserts the
- * symbol-dict invariant on the wire.
+ * The modes differ only in where the catch-up's dictionary comes from: memory
+ * mode keeps it in an in-process mirror; file-backed store-and-forward keeps it in
+ * a per-slot {@code .symbol-dict} file so a recovered or orphan-drained slot (a
+ * fresh process with no in-memory mirror) can rebuild it. This test asserts the
+ * monotonic wire framing in both modes and the presence of that dictionary file.
  */
 public class SelfSufficientFramesTest {
 
@@ -56,58 +64,487 @@ public class SelfSufficientFramesTest {
     private static final int DELTA_START_OFFSET = 12;
 
     @Test
-    public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception {
-        // Send two batches against the same connection, each with a
-        // distinct symbol value. With the old schema-ref/delta encoding,
-        // batch 2 would emit deltaStart=1, deltaCount=1 — only the new
-        // symbol. With self-sufficient frames, batch 2 must emit
-        // deltaStart=0 covering BOTH symbols.
-        CapturingHandler handler = new CapturingHandler();
-        try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
-            int port = server.getPort();
-            server.start();
-            Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
-
-            try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
-                sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
-                sender.flush();
-                waitFor(() -> handler.batches.size() >= 1, 5_000);
-
-                sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
-                sender.flush();
-                waitFor(() -> handler.batches.size() >= 2, 5_000);
+    public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception {
+        // File-backed SF also ships monotonic deltas now: batch 2 carries only
+        // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict
+        // so a recovered/orphan-drained slot can rebuild it.
+        Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient");
+        try {
+            assertMemoryLeak(() -> {
+                CapturingHandler handler = new CapturingHandler();
+                try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                    int port = server.getPort();
+                    server.start();
+                    Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                    String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                    // The engine places slot files under sf_dir/ (default "default").
+                    Path dictFile = sfDir.resolve("default").resolve(".symbol-dict");
+                    try (Sender sender = Sender.fromConfig(config)) {
+                        sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
+                        sender.flush();
+                        waitFor(() -> handler.batches.size() >= 1, 5_000);
+
+                        sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
+                        sender.flush();
+                        waitFor(() -> handler.batches.size() >= 2, 5_000);
+
+                        // Check the persisted dictionary while the sender is live: a
+                        // fully-drained close intentionally unlinks it (slot cleanup).
+                        Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile));
+                        byte[] dict = Files.readAllBytes(dictFile);
+                        Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha"));
+                        Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta"));
+                    }
+
+                    Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
+                    byte[] b1 = handler.batches.get(0);
+                    byte[] b2 = handler.batches.get(1);
+
+                    Assert.assertEquals("batch 1 deltaStart must be 0",
+                            0, readVarint(b1, DELTA_START_OFFSET));
+                    Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1));
+                    // batch 2 ships ONLY beta as a delta from id 1.
+                    Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)",
+                            1, readVarint(b2, DELTA_START_OFFSET));
+                    Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)",
+                            1, readVarint(b2, DELTA_START_OFFSET + 1));
+                }
+            });
+        } finally {
+            rmDir(sfDir);
+        }
+    }
+
+    @Test
+    public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Exception {
+        // When the per-slot .symbol-dict cannot be opened in disk mode,
+        // isDeltaDictEnabled() is false and the sender must fall back to
+        // self-sufficient frames: every batch re-ships the WHOLE dictionary from
+        // id 0. A recovered / orphan-drained slot then has no dictionary to
+        // rebuild deltas from, so a monotonic delta would dangle ids on the fresh
+        // server -- the full-dict frame is the safe degradation. Force the open
+        // failure by planting a DIRECTORY where the dictionary file belongs:
+        // openRW / openCleanRW on a directory fails, so open() returns null.
+        Path sfDir = Files.createTempDirectory("qwp-sf-fallback");
+        Path dictPath = sfDir.resolve("default").resolve(".symbol-dict");
+        Files.createDirectories(dictPath);             // a directory, not a file
+        Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd
+        try {
+            assertMemoryLeak(() -> {
+                CapturingHandler handler = new CapturingHandler();
+                try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                    int port = server.getPort();
+                    server.start();
+                    Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                    String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                    try (Sender sender = Sender.fromConfig(config)) {
+                        sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
+                        sender.flush();
+                        waitFor(() -> handler.batches.size() >= 1, 5_000);
+
+                        sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
+                        sender.flush();
+                        waitFor(() -> handler.batches.size() >= 2, 5_000);
+                    }
+
+                    // The planted directory is untouched -- the dictionary never
+                    // opened, so delta encoding stayed disabled.
+                    Assert.assertTrue("planted .symbol-dict directory must remain (open failed)",
+                            Files.isDirectory(dictPath));
+
+                    Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
+                    byte[] b1 = handler.batches.get(0);
+                    byte[] b2 = handler.batches.get(1);
+
+                    // Full-dict fallback: BOTH batches start at id 0, and batch 2
+                    // re-ships the WHOLE dictionary (alpha + beta), NOT a monotonic
+                    // delta (which would be deltaStart=1, deltaCount=1 as above).
+                    Assert.assertEquals("batch 1 deltaStart must be 0",
+                            0, readVarint(b1, DELTA_START_OFFSET));
+                    Assert.assertEquals("batch 1 deltaCount must be 1",
+                            1, readVarint(b1, DELTA_START_OFFSET + 1));
+                    Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)",
+                            0, readVarint(b2, DELTA_START_OFFSET));
+                    Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)",
+                            2, readVarint(b2, DELTA_START_OFFSET + 1));
+                }
+            });
+        } finally {
+            rmDir(sfDir);
+        }
+    }
+
+    private static boolean containsUtf8(byte[] haystack, String needle) {
+        byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        outer:
+        for (int i = 0; i + n.length <= haystack.length; i++) {
+            for (int j = 0; j < n.length; j++) {
+                if (haystack[i + j] != n[j]) {
+                    continue outer;
+                }
+            }
+            return true;
+        }
+        return false;
+    }
+
+    @Test
+    public void testMemoryModeShipsMonotonicDelta() throws Exception {
+        // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries
+        // only "beta" as a delta starting at id 1, not the whole dictionary.
+        assertMemoryLeak(() -> {
+            CapturingHandler handler = new CapturingHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+                    sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.batches.size() >= 1, 5_000);
+
+                    sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.batches.size() >= 2, 5_000);
+                }
+
+                Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
+                byte[] b1 = handler.batches.get(0);
+                byte[] b2 = handler.batches.get(1);
+
+                // Batch 1 introduces alpha at id 0.
+                Assert.assertEquals("batch 1 deltaStart must be 0",
+                        0, readVarint(b1, DELTA_START_OFFSET));
+                Assert.assertEquals("batch 1 deltaCount must be 1",
+                        1, readVarint(b1, DELTA_START_OFFSET + 1));
+
+                // Batch 2 ships ONLY beta as a delta from id 1.
+                Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)",
+                        1, readVarint(b2, DELTA_START_OFFSET));
+                Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)",
+                        1, readVarint(b2, DELTA_START_OFFSET + 1));
+            }
+        });
+    }
+
+    @Test
+    public void testSplitBatchShipsDeltaOnFirstFrameOnly() throws Exception {
+        // A single flush whose encoded size exceeds the server's batch cap is
+        // split into one frame per table (flushPendingRowsSplit). The FIRST split
+        // frame carries the whole batch's symbol-dict delta and advances the
+        // baseline; the remaining frames carry an EMPTY delta and only reference
+        // ids the first frame already registered. A regression that shipped each
+        // table's own symbols (wrong deltaStart) would dangle ids on the server.
+        assertMemoryLeak(() -> {
+            CapturingHandler handler = new CapturingHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                // Padding inflates each table past half the cap, so the combined
+                // two-table message exceeds it while each single-table frame fits.
+                String pad = new String(new char[60]).replace('\0', 'x');
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+                    // Buffer TWO tables (symbols "a" id 0, "b" id 1), then ONE flush.
+                    sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow();
+                    sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow();
+                    sender.flush();
+                    waitFor(() -> handler.batches.size() >= 2, 5_000);
+                }
+
+                Assert.assertEquals("the oversized two-table batch must split into 2 frames",
+                        2, handler.batches.size());
+                byte[] f1 = handler.batches.get(0);
+                byte[] f2 = handler.batches.get(1);
+
+                // First split frame ships the whole batch's dictionary (a + b).
+                Assert.assertEquals("first split frame deltaStart must be 0",
+                        0, readVarint(f1, DELTA_START_OFFSET));
+                Assert.assertEquals("first split frame ships both new symbols",
+                        2, readVarint(f1, DELTA_START_OFFSET + 1));
+                // Second split frame carries an empty delta above the advanced baseline.
+                Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)",
+                        2, readVarint(f2, DELTA_START_OFFSET));
+                Assert.assertEquals("second split frame carries no new symbols",
+                        0, readVarint(f2, DELTA_START_OFFSET + 1));
             }
+        });
+    }
+
+    @Test
+    public void testOversizedTableSplitStrandsNothing() throws Exception {
+        // Regression: flushPendingRowsSplit publishes each table's frame one at a
+        // time (all but the last deferred, i.e. appended but uncommitted). If a LATER
+        // table's frame exceeds the cap, the split must not have already published an
+        // earlier table's frame -- otherwise that prefix strands on the ring, a later
+        // commit delivers it as a partial batch, and resetTableBuffersAfterFlush
+        // discards every source row, all while flush() reported failure to the
+        // caller. The pre-flight size pass makes the split all-or-nothing: an
+        // oversized table throws BEFORE any frame is published. Pre-fix, the "small"
+        // table's frame was published and committed on close, so the server saw it.
+        assertMemoryLeak(() -> {
+            CapturingHandler handler = new CapturingHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                server.setAdvertisedMaxBatchSize(200);
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                // auto_flush_bytes=off lets "big" accumulate PAST the cap (byte-based
+                // auto-flush is otherwise clamped to 90% of the cap and would flush
+                // first); the row/interval limits are set high so nothing auto-flushes
+                // during the test. Each row stays under the per-row guard (< cap), but
+                // 12 rows make "big"'s single frame exceed the cap, which no split can
+                // shrink. "small" (added first) fits; "big" (added second) does not, so
+                // the split hits it AFTER publishing "small" pre-fix.
+                String pad = new String(new char[40]).replace('\0', 'x');
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+                        + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) {
+                    sender.table("small").symbol("s", "a").longColumn("v", 1L).atNow();
+                    for (int i = 0; i < 12; i++) {
+                        sender.table("big").stringColumn("p", pad).longColumn("v", (long) i).atNow();
+                    }
+                    try {
+                        sender.flush();
+                        Assert.fail("an oversized single-table split frame must throw");
+                    } catch (LineSenderException e) {
+                        Assert.assertTrue(e.getMessage(),
+                                e.getMessage().contains("too large for server batch cap"));
+                    }
+                    // Sender.flush() is retryable: a rejected flush must leave the
+                    // source rows intact. Repeating it under the same cap therefore
+                    // has to reject the same batch again. Before the fix, the first
+                    // failure reset every table buffer and this second flush was a
+                    // silent no-op -- the caller's rows had been lost.
+                    try {
+                        sender.flush();
+                        Assert.fail("retrying the retained oversized batch must throw again");
+                    } catch (LineSenderException e) {
+                        Assert.assertTrue(e.getMessage(),
+                                e.getMessage().contains("too large for server batch cap"));
+                    }
+                    // Explicit reset is the caller-controlled discard boundary and
+                    // prevents close() from making a third retry in this test.
+                    sender.reset();
+                    // close() drains the ring: pre-fix, the stranded "small" frame
+                    // would be sent (and committed) here.
+                }
+
+                // No DATA frame (tableCount > 0) may have reached the server: the
+                // oversized-table split published nothing. Pre-fix, "small" arrived.
+                long dataFrames = 0;
+                for (byte[] frame : handler.batches) {
+                    if (frame.length >= 8 && (((frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8)) > 0)) {
+                        dataFrames++;
+                    }
+                }
+                Assert.assertEquals("an oversized-table split must publish NO data frame -- an "
+                                + "earlier table's frame must not strand on the ring", 0, dataFrames);
+            }
+        });
+    }
+
+    @Test
+    public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception {
+        // The SPLIT path's write-ahead ordering: persistNewSymbolsBeforePublish must run
+        // BEFORE sealAndSwapBuffer publishes the frame to the ring, exactly as it does on
+        // the non-split path.
+        //
+        // Its sibling below (testFileModeSplitPersistsDictBeforePublish) checks the dict
+        // after a SUCCESSFUL flush, which proves nothing about ORDER -- the symbols land
+        // in the file either way. Only a publish that FAILS while the symbols are new can
+        // tell the two apart: with the write-ahead intact the symbols are already durable
+        // when the publish throws; with the persist moved after the publish, the throw
+        // beats it and the dictionary is still empty. (The non-split path is pinned this
+        // way by DeltaDictRecoveryTest.testFailedPublishDoesNotDuplicatePersistedSymbols;
+        // the split path had no equivalent.)
+        //
+        // Why it matters: store-and-forward is process-crash durable. If a split frame
+        // reaches the ring before its symbols reach .symbol-dict, a JVM crash in that
+        // window leaves a recorded frame whose deltaStart exceeds the recovered
+        // dictionary -- so recovery declares the slot unreplayable and quarantines a slot
+        // that should have drained cleanly.
+        //
+        // Setup: the server cap (150) splits the two-table batch, and each split frame
+        // still FITS that cap, so the split pre-flight passes and the publish is actually
+        // attempted. sf_max_bytes=64 then makes every frame larger than the segment, so
+        // appendBlocking fails with PAYLOAD_TOO_LARGE -- deterministically, no
+        // backpressure timing needed.
+        Path sfDir = Files.createTempDirectory("qwp-sf-split-persist-fail");
+        try {
+            assertMemoryLeak(() -> {
+                CapturingHandler handler = new CapturingHandler();
+                try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                    server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+                    int port = server.getPort();
+                    server.start();
+                    Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                    String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                            + ";sf_max_bytes=64;";
+                    String pad = TestUtils.repeat("x", 60);
+                    Sender sender = Sender.fromConfig(config);
+                    try {
+                        sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow();
+                        sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow();
+                        try {
+                            sender.flush();
+                            Assert.fail("a split frame larger than the segment must fail to publish");
+                        } catch (LineSenderException expected) {
+                            // PAYLOAD_TOO_LARGE -- the publish, not the pre-flight.
+                        }
+                        Assert.assertEquals("the publish must fail, so no frame reaches the server",
+                                0, handler.batches.size());
 
-            Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
-            byte[] b1 = handler.batches.get(0);
-            byte[] b2 = handler.batches.get(1);
-
-            // The deltaStart varint sits right after the 12-byte header.
-            // For self-sufficient frames it must be 0 (single byte 0x00)
-            // in BOTH batches — regardless of how many symbols the prior
-            // batch already shipped.
-            int deltaStart1 = readVarint(b1, DELTA_START_OFFSET);
-            int deltaStart2 = readVarint(b2, DELTA_START_OFFSET);
-            Assert.assertEquals("batch 1 deltaStart must be 0", 0, deltaStart1);
-            Assert.assertEquals("batch 2 deltaStart must be 0 (self-sufficient)",
-                    0, deltaStart2);
-
-            // batch 2 must include >= 2 symbols in its delta dict (alpha
-            // from the prior batch + beta from this one). The varint at
-            // DELTA_START_OFFSET+1 is deltaCount.
-            int deltaCount2 = readVarint(b2, DELTA_START_OFFSET + 1);
-            Assert.assertTrue("batch 2 must redefine at least 2 symbols, got " + deltaCount2,
-                    deltaCount2 >= 2);
-
-            // Sanity: batch 2 should NOT be much smaller than batch 1 —
-            // with schema-ref/delta encoding it would have been; with
-            // self-sufficient frames the size is in the same ballpark.
-            Assert.assertTrue("batch 2 (" + b2.length + " bytes) must not be drastically smaller than batch 1 ("
-                            + b1.length + ")",
-                    b2.length >= b1.length / 2);
+                        // The write-ahead already ran: both of the batch's new symbols are
+                        // durable even though the frame that references them never
+                        // published. Move the persist after sealAndSwapBuffer and this is 0.
+                        ObjList persisted =
+                                ((QwpWebSocketSender) sender).getPersistedSymbolsForTest();
+                        Assert.assertNotNull(persisted);
+                        Assert.assertEquals("the split path must persist its new symbols BEFORE "
+                                + "publishing the frame that references them", 2, persisted.size());
+                        Assert.assertEquals("alpha", persisted.getQuick(0));
+                        Assert.assertEquals("bravo", persisted.getQuick(1));
+                    } finally {
+                        try {
+                            sender.close();
+                        } catch (LineSenderException ignored) {
+                            // close() re-flushes the still-buffered oversized rows and fails
+                            // again; expected, and not what this test asserts. close() still
+                            // runs its resource cleanup, so no native memory leaks.
+                        }
+                    }
+                }
+            });
+        } finally {
+            rmDir(sfDir);
         }
     }
 
+    @Test
+    public void testFileModeSplitPersistsDictBeforePublish() throws Exception {
+        // File-mode store-and-forward + a SPLIT flush: a two-table batch whose combined
+        // size exceeds the server cap splits into one frame per table
+        // (flushPendingRowsSplit). The first split frame's write-ahead persist
+        // (persistNewSymbolsBeforePublish, the appendRawEntries fast path) must record
+        // the batch's new symbols in .symbol-dict BEFORE the frames publish, so a
+        // recovered / orphan-drained slot can rebuild what the delta frames reference.
+        // The other split tests run in memory mode, so this is the only coverage of the
+        // split x persist path with a live PersistedSymbolDict.
+        Path sfDir = Files.createTempDirectory("qwp-sf-split-persist");
+        try {
+            assertMemoryLeak(() -> {
+                CapturingHandler handler = new CapturingHandler();
+                try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                    server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+                    int port = server.getPort();
+                    server.start();
+                    Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                    String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+                    Path dictFile = sfDir.resolve("default").resolve(".symbol-dict");
+                    String pad = new String(new char[60]).replace('\0', 'x');
+                    try (Sender sender = Sender.fromConfig(config)) {
+                        // Two tables, two new symbols, ONE flush -> the combined message
+                        // exceeds cap 150 and splits into two frames.
+                        sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow();
+                        sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow();
+                        sender.flush();
+                        waitFor(() -> handler.batches.size() >= 2, 5_000);
+
+                        // Check .symbol-dict while the sender is live: a fully-drained
+                        // close would unlink it. The split's first-frame write-ahead
+                        // persist must have recorded BOTH new symbols.
+                        Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile));
+                        byte[] dict = Files.readAllBytes(dictFile);
+                        Assert.assertTrue("split-flush persist must record alpha", containsUtf8(dict, "alpha"));
+                        Assert.assertTrue("split-flush persist must record bravo", containsUtf8(dict, "bravo"));
+                    }
+
+                    Assert.assertEquals("the oversized two-table batch must split into 2 frames",
+                            2, handler.batches.size());
+                }
+            });
+        } finally {
+            rmDir(sfDir);
+        }
+    }
+
+    @Test
+    public void testSplitPreflightAdvancesBaselineSoLaterFramesArentSizedWithTheDelta() throws Exception {
+        // Regression for the split pre-flight baseline advance in flushPendingRowsSplit
+        // (the "Mirror advanceSentMaxSymbolId" step). Only the FIRST split frame ships
+        // the batch's symbol-dict delta; the rest ship an EMPTY delta and reference ids
+        // the first frame already registered. The pre-flight size pass must advance
+        // simBaseline after the first table so it STOPS adding combinedDeltaEntriesLen
+        // to the later frames' estimated sizes. Without that advance, a later table
+        // whose real (empty-delta) frame fits the cap is mis-estimated as still carrying
+        // the whole delta and wrongly rejected with "single table batch too large" --
+        // discarding a legitimately shippable batch (fail-closed data loss).
+        //
+        // Shape (memory mode, delta enabled): a LARGE combined delta (two ~64-char
+        // symbols) rides only the first split frame. The first table (added first, so
+        // the first split frame) has a tiny body, so delta + body1 fits the cap. The
+        // second table has a big body: body2 alone fits the cap, but delta + body2 does
+        // NOT. The real code splits and ships both frames; the un-advanced pre-flight
+        // would size the second frame WITH the delta and throw. Table order is
+        // insertion order (CharSequenceObjHashMap.keys()), so t1 is the delta frame.
+        assertMemoryLeak(() -> {
+            CapturingHandler handler = new CapturingHandler();
+            try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+                server.setAdvertisedMaxBatchSize(200);
+                int port = server.getPort();
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                // Two long symbols => a large combined delta section (~130 bytes) that
+                // rides ONLY the first split frame. The symbol STRINGS live in the delta,
+                // not in either table body (the body carries only the varint global id).
+                String longSymA = new String(new char[64]).replace('\0', 'a');
+                String longSymB = new String(new char[64]).replace('\0', 'b');
+                String bigPad = new String(new char[100]).replace('\0', 'x');
+                // auto_flush off so both rows batch into one flush (byte-based auto-flush
+                // is otherwise clamped under the cap and would flush before the split).
+                try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+                        + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) {
+                    // t1 (added first -> first split frame): carries the whole delta but a
+                    // tiny body, so delta + body1 fits the 200-byte cap.
+                    sender.table("t1").symbol("s", longSymA).longColumn("v", 1L).atNow();
+                    // t2 (second split frame): empty delta but a big body. body2 alone
+                    // fits the cap; delta + body2 does NOT -- the mis-size the advance
+                    // prevents.
+                    sender.table("t2").symbol("s", longSymB).stringColumn("p", bigPad).longColumn("v", 2L).atNow();
+                    // Must NOT throw: with the baseline advanced, t2's frame is sized
+                    // WITHOUT the delta and fits. A broken advance throws "too large" here.
+                    sender.flush();
+                    waitFor(() -> handler.batches.size() >= 2, 5_000);
+                }
+
+                Assert.assertEquals("the batch must split into 2 frames, neither spuriously rejected",
+                        2, handler.batches.size());
+                byte[] f1 = handler.batches.get(0);
+                byte[] f2 = handler.batches.get(1);
+                // First split frame ships the whole delta (both new symbols, ids 0 and 1).
+                Assert.assertEquals("first split frame deltaStart must be 0",
+                        0, readVarint(f1, DELTA_START_OFFSET));
+                Assert.assertEquals("first split frame ships both new symbols",
+                        2, readVarint(f1, DELTA_START_OFFSET + 1));
+                // Second split frame carries an EMPTY delta above the advanced baseline --
+                // the whole point: it is not re-sized (or re-sent) with the delta.
+                Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)",
+                        2, readVarint(f2, DELTA_START_OFFSET));
+                Assert.assertEquals("second split frame carries no new symbols",
+                        0, readVarint(f2, DELTA_START_OFFSET + 1));
+            }
+        });
+    }
+
     private static int readVarint(byte[] buf, int offset) {
         // Simple unsigned varint decode — sufficient for small values.
         int result = 0;
@@ -122,6 +559,10 @@ private static int readVarint(byte[] buf, int offset) {
         throw new IllegalStateException("varint truncated");
     }
 
+    private static void rmDir(Path dir) {
+        TestUtils.removeTmpDirRec(dir == null ? null : dir.toString());
+    }
+
     private static void waitFor(BoolCondition cond, long timeoutMillis) {
         long deadline = System.currentTimeMillis() + timeoutMillis;
         while (System.currentTimeMillis() < deadline) {
@@ -157,6 +598,7 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat
             }
         }
 
+        // Mirrors WebSocketResponse STATUS_OK layout: status u8 | sequence u64 | table_count u16
         static byte[] buildAck(long seq) {
             byte[] buf = new byte[1 + 8 + 2];
             ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java
index 0a6b69f8..82885ba7 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java
@@ -241,8 +241,8 @@ public void testTerminalUpgradeMarksFailedImmediately() throws Exception {
             CountingListener listener = new CountingListener();
             // A genuinely non-retriable upgrade error (non-421 5xx upgrade reject) is
             // terminal -- waiting will not fix it -- so the drainer quarantines on the
-            // first attempt, exactly like the live sender's background loop halts on
-            // auth/upgrade. A TRANSPORT error, by contrast, is transient and is
+            // first attempt under the orphan reconnect policy. A TRANSPORT error,
+            // by contrast, is transient and is
             // retried (see testTransportErrorNeverQuarantinesInvariantB).
             ScriptedFactory factory = ScriptedFactory.alwaysFailing(
                     () -> new WebSocketUpgradeException(500, null, "server error during upgrade"));
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
index f4de1ff2..0bdddb3e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
@@ -31,6 +31,7 @@
 import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock;
 import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.ObjList;
 import io.questdb.client.std.Unsafe;
@@ -297,6 +298,51 @@ public void testRecoveryAdvancesAckedFsnPastWatermark() throws Exception {
         });
     }
 
+    @Test
+    public void testV2DiskSlotForcesLegacyReaderToFailClosed() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+            try {
+                try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) {
+                    assertEquals(0L, engine.appendBlocking(buf, 16));
+                }
+
+                // Real segments carry v2: their payload may depend on the
+                // persisted dictionary and must never be replayed by a v1-only
+                // client as though every frame were self-sufficient.
+                try (MmapSegment data = MmapSegment.openExisting(tmpDir + "/sf-initial.sfa")) {
+                    assertEquals(MmapSegment.VERSION, data.version());
+                    assertEquals(1L, data.frameCount());
+                }
+
+                // A legacy reader rejects the v2 data segment, but accepts both
+                // guards as ordinary non-empty v1 segments. Their ranges cannot
+                // be contiguous, so its mandatory global recovery check throws
+                // before any replay or fresh-slot fallback can happen.
+                try (MmapSegment guardA = MmapSegment.openExisting(
+                        tmpDir + "/.qwp-v2-guard-a.sfa");
+                     MmapSegment guardB = MmapSegment.openExisting(
+                             tmpDir + "/.qwp-v2-guard-b.sfa")) {
+                    assertEquals(MmapSegment.LEGACY_VERSION, guardA.version());
+                    assertEquals(MmapSegment.LEGACY_VERSION, guardB.version());
+                    assertEquals(1L, guardA.frameCount());
+                    assertEquals(1L, guardB.frameCount());
+                    assertTrue("legacy guard ranges must force an FSN gap",
+                            guardA.baseSeq() + guardA.frameCount() != guardB.baseSeq());
+                }
+
+                // Current recovery skips the reserved guards and adopts only
+                // the real v2 log.
+                try (CursorSendEngine recovered = new CursorSendEngine(tmpDir, 4096)) {
+                    assertTrue(recovered.wasRecoveredFromDisk());
+                    assertEquals(0L, recovered.publishedFsn());
+                }
+            } finally {
+                Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testRecoveryIgnoresWatermarkAbovePublishedFsn() throws Exception {
         // Pin the corruption defence: a watermark higher than what the
@@ -412,7 +458,11 @@ public void testRestartIntoNonEmptySfDirContinuesFsnSequence() throws Exception
                     int rc = 1;
                     while (rc > 0) {
                         String name = Files.utf8ToString(Files.findName(find));
-                        if (name != null && name.endsWith(".sfa")) sfaCount++;
+                        if (name != null
+                                && name.endsWith(".sfa")
+                                && !name.startsWith(".qwp-v2-guard-")) {
+                            sfaCount++;
+                        }
                         rc = Files.findNext(find);
                     }
                 } finally {
@@ -608,11 +658,12 @@ private static void assertSlotCanBeReacquired(String sfDir) {
     private static Throwable invokeOwnedPrivateConstructorExpectingFailure(
             String sfDir, long segmentSizeBytes, SegmentManager manager) throws Exception {
         Constructor ctor = CursorSendEngine.class.getDeclaredConstructor(
-                String.class, long.class, SegmentManager.class, boolean.class, long.class);
+                String.class, long.class, SegmentManager.class, boolean.class, long.class,
+                FilesFacade.class);
         ctor.setAccessible(true);
         try {
             ctor.newInstance(sfDir, segmentSizeBytes, manager, true,
-                    CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS);
+                    CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, FilesFacade.INSTANCE);
             fail("expected constructor failure");
             return null;
         } catch (InvocationTargetException e) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
new file mode 100644
index 00000000..86ba58a0
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
@@ -0,0 +1,1069 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.DefaultHttpClientConfiguration;
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException;
+import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.network.PlainSocketFactory;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in
+ * {@link CursorWebSocketSendLoop#setWireBaselineWithCatchUp}.
+ * 

+ * On a fresh connection the loop re-registers the whole dictionary with a
+ * catch-up frame BEFORE replaying data frames. Each catch-up frame consumes a
+ * wire sequence, so the loop anchors {@code fsnAtZero = replayStart - catchUpFrames}
+ * to keep every catch-up frame mapped to an already-acked FSN. Dropping the
+ * {@code - catchUpFrames} term is silent data loss: a server ACK for a catch-up
+ * frame then translates through {@code engine.acknowledge(fsnAtZero + wireSeq)}
+ * to an FSN at or above {@code replayStart}, trimming a not-yet-delivered data
+ * frame from the store-and-forward log.
+ * 

+ * The loop is constructed but never {@link CursorWebSocketSendLoop#start started};
+ * the catch-up runs against a stub {@link WebSocketClient} that counts frames, and
+ * the OK is delivered straight into the inner {@code ResponseHandler} -- the same
+ * white-box idiom {@code CursorWebSocketSendLoopDurableAckTest} uses, because
+ * {@code setWireBaselineWithCatchUp} and the wire ports have no public entry point.
+ * {@link CursorSendEngine#ackedFsn()} is the authoritative trim watermark asserted
+ * against.
+ */
+public class CursorWebSocketSendLoopCatchUpAlignmentTest {
+
+    private String tmpDir;
+
+    @Before
+    public void setUp() {
+        tmpDir = TestUtils.createTmpDir("qdb-cursor-catchup-");
+    }
+
+    @After
+    public void tearDown() {
+        TestUtils.removeTmpDir(tmpDir);
+    }
+
+    @Test
+    public void testCatchUpFrameAckDoesNotAdvanceTrimWatermark() throws Exception {
+        // Single catch-up frame (server advertises no cap). Two frames were
+        // acked before the reconnect (ackedFsn=1), FSN 2 is unacked. The catch-up
+        // frame's OK must NOT advance the watermark past 1 -- it carries no data,
+        // only the dictionary the fresh server needs before replay.
+        TestUtils.assertMemoryLeak(() -> {
+            CatchUpCapturingClient client = new CatchUpCapturingClient(0); // 0 => no cap => one frame
+            try (CursorSendEngine engine = newEngine()) {
+                appendFrames(engine, 3);            // FSN 0,1,2 published
+                engine.acknowledge(1);              // ackedFsn=1 => replayStart=2, FSN 2 still unacked
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    seedMirror(loop, "s0", "s1");   // sentDictCount=2 => catch-up fires
+                    long replayStart = engine.ackedFsn() + 1L; // = 2
+
+                    invokeSetWireBaselineWithCatchUp(loop, replayStart);
+
+                    assertEquals("whole dictionary fits one frame under no cap",
+                            1, client.framesSent);
+
+                    // Behavioural (the harm): the catch-up frame (wire seq 0) is
+                    // OK'd by the fresh server. It carries no data, so it must
+                    // resolve to an already-acked FSN and leave the trim watermark
+                    // untouched -- advancing it would trim the undelivered FSN 2.
+                    deliverOk(loop, 0);
+                    assertEquals("catch-up frame ACK must not advance the trim watermark "
+                                    + "(would trim an undelivered data frame -> silent data loss)",
+                            1L, engine.ackedFsn());
+                    // Mechanism: the catch-up frames are anchored below replayStart.
+                    assertEquals("fsnAtZero must be anchored catchUpFrames below replayStart",
+                            replayStart - client.framesSent, readLong(loop, "fsnAtZero"));
+                } finally {
+                    loop.close(); // frees the seeded mirror + the stub client's buffers
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Exception {
+        // A small advertised cap splits the dictionary across several catch-up
+        // frames, so the fsnAtZero offset must subtract the full frame count. Ack
+        // the LAST catch-up wire sequence: it still maps below replayStart. With
+        // the offset dropped it would translate to replayStart+1 and over-trim.
+        TestUtils.assertMemoryLeak(() -> {
+            CatchUpCapturingClient client = new CatchUpCapturingClient(40); // budget 12 => one 11-byte symbol per frame
+            try (CursorSendEngine engine = newEngine()) {
+                appendFrames(engine, 5);            // FSN 0..4 published
+                engine.acknowledge(2);              // ackedFsn=2 => replayStart=3, FSN 3,4 unacked
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    seedMirror(loop, "symbol0000", "symbol0001"); // 11 bytes each -> two frames
+                    long replayStart = engine.ackedFsn() + 1L; // = 3
+
+                    invokeSetWireBaselineWithCatchUp(loop, replayStart);
+
+                    assertEquals("cap must split the two symbols across two frames",
+                            2, client.framesSent);
+
+                    // ACK the highest catch-up wire sequence (the last catch-up
+                    // frame). It too must map below replayStart -- with the offset
+                    // dropped it translates to replayStart+1 and over-trims.
+                    deliverOk(loop, client.framesSent - 1);
+                    assertEquals("no catch-up frame ACK may advance the trim watermark",
+                            2L, engine.ackedFsn());
+                    assertEquals("fsnAtZero must subtract the full split frame count",
+                            replayStart - client.framesSent, readLong(loop, "fsnAtZero"));
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testSplitCatchUpReusesOneFrameBufferAcrossReconnects() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            CatchUpCapturingClient client = new CatchUpCapturingClient(3_100);
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    seedMirror(loop, TestUtils.repeat("x", 3_000), TestUtils.repeat("y", 3_000));
+
+                    invokeSetWireBaselineWithCatchUp(loop, 0L);
+                    assertEquals("the small cap must split the dictionary", 2, client.framesSent);
+                    assertEquals("the split chunks fit the initial native buffer",
+                            1, loop.catchUpFrameGrowthCount());
+
+                    client.cap = 7_000;
+                    invokeSetWireBaselineWithCatchUp(loop, 0L);
+                    assertEquals("the larger cap must combine the dictionary", 3, client.framesSent);
+                    assertEquals("the combined frame must grow the retained native buffer once",
+                            2, loop.catchUpFrameGrowthCount());
+
+                    invokeSetWireBaselineWithCatchUp(loop, 0L);
+                    assertEquals("the next reconnect sends one combined frame", 4, client.framesSent);
+                    assertEquals("the grown native frame buffer must be reused across reconnects",
+                            2, loop.catchUpFrameGrowthCount());
+                } finally {
+                    // assertMemoryLeak verifies that close releases the retained buffer.
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Exception {
+        // A transient wire failure WHILE shipping the catch-up (the fresh
+        // connection drops mid-handshake) must surface as a retriable
+        // CatchUpSendException for the reconnect loop to handle -- it must NOT
+        // call fail(). From inside the catch-up fail() re-enters connectLoop
+        // (corrupting the fsnAtZero/nextWireSeq mapping, or overflowing the stack
+        // on a flapping connection) or, with no reconnect attempt reachable,
+        // latches a terminal -- turning a transient outage into a hard failure and
+        // breaking store-and-forward. Only the oversized-entry (non-retriable)
+        // terminal was covered; this pins the retriable path.
+        TestUtils.assertMemoryLeak(() -> {
+            CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); // sendBinary throws
+            try (CursorSendEngine engine = newEngine()) {
+                appendFrames(engine, 2);
+                engine.acknowledge(0); // ackedFsn=0 => a real unacked frame exists behind the catch-up
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    seedMirror(loop, "s0", "s1"); // non-empty dict => catch-up fires and hits the failing send
+                    try {
+                        invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                        fail("a transient catch-up send failure must raise a retriable "
+                                + "CatchUpSendException, not be swallowed into fail()/a terminal");
+                    } catch (InvocationTargetException e) {
+                        assertEquals("transient catch-up send failure must surface as CatchUpSendException",
+                                "CatchUpSendException", e.getCause().getClass().getSimpleName());
+                    }
+                    // Retriable, not terminal: the producer-facing error latch stays clear.
+                    loop.checkError();
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testAccumulateSentDictPartialOverlapExtendsMirror() throws Exception {
+        // M3: accumulateSentDict must handle a delta that STRADDLES the mirror tip
+        // (deltaStart < sentDictCount < deltaStart+deltaCount) by copying only the
+        // new tail, not dropping the whole frame. The monotonic producer never emits
+        // a straddling delta in steady state (so the pre-fix drop-whole-frame guard
+        // passed every test), but a torn-dict replay can seed the mirror smaller than
+        // a frame's coverage. Seed the mirror with 1 symbol, feed a [0..2] delta, and
+        // assert the mirror extends to all 3 -- pre-fix it stayed at 1, leaving the
+        // reconnect catch-up incomplete and shifting server-side ids.
+        TestUtils.assertMemoryLeak(() -> {
+            CatchUpCapturingClient client = new CatchUpCapturingClient(0);
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    seedMirror(loop, "aa"); // sentDictCount = 1, mirror holds "aa"
+                    int[] frameLen = new int[1];
+                    long frame = buildDeltaFrame(0, new String[]{"aa", "bb", "cc"}, frameLen);
+                    try {
+                        Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(
+                                "accumulateSentDict", long.class, int.class, int.class);
+                        m.setAccessible(true);
+                        m.invoke(loop, frame, frameLen[0], 0);
+                    } finally {
+                        Unsafe.free(frame, frameLen[0], MemoryTag.NATIVE_DEFAULT);
+                    }
+                    assertEquals("straddling delta must extend the mirror to all 3 ids",
+                            3, readInt(loop, "sentDictCount"));
+                    assertEquals("mirror must hold the two new tail symbols after the "
+                                    + "already-held prefix, gap-free",
+                            Arrays.asList("aa", "bb", "cc"), readMirrorSymbols(loop));
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception {
+        // M3: sendDictCatchUp caps each chunk under the budget, so the single-frame
+        // catch-up path cannot overflow its int frameLen at any real cardinality. The
+        // guard must still be LOCAL -- a future caller must not be able to feed a
+        // wrapped-negative frameLen to Unsafe.malloc. An oversized symbolsLen must
+        // fail loud (CatchUpSendException) BEFORE the malloc; the guard fires before
+        // symbolsAddr is read, so a dummy address is fine.
+        TestUtils.assertMemoryLeak(() -> {
+            CatchUpCapturingClient client = new CatchUpCapturingClient(0);
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(
+                            "sendCatchUpChunk", int.class, int.class, long.class, int.class);
+                    m.setAccessible(true);
+                    // symbolsLen past the mirror ceiling: HEADER + varints + symbolsLen
+                    // overflows an int, so the guard must reject it before malloc.
+                    m.invoke(loop, 0, 1, 0L, Integer.MAX_VALUE - 4);
+                    fail("an overflowing catch-up frame size must fail loud, not malloc negative");
+                } catch (InvocationTargetException e) {
+                    assertEquals("overflow must surface as CatchUpSendException",
+                            "CatchUpSendException", e.getCause().getClass().getSimpleName());
+                    assertTrue("message must name the frame-size guard: " + e.getCause().getMessage(),
+                            e.getCause().getMessage().contains("catch-up frame exceeds the maximum size"));
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testForegroundCatchUpCapGapRetriesPastOrphanBudget() throws Exception {
+        // The foreground policy must never accrue or exhaust the orphan drainer's
+        // quarantine budget. Drive more cap gaps than that entire budget and assert every
+        // failure remains retriable to the I/O loop and invisible to the producer.
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+            maxField.setAccessible(true);
+            int maxAttempts = maxField.getInt(null);
+            CatchUpCapturingClient client = new CatchUpCapturingClient(160);
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newForegroundLoop(engine, client);
+                try {
+                    seedMirror(loop, TestUtils.repeat("x", 200));
+                    for (int i = 1; i <= maxAttempts + 4; i++) {
+                        try {
+                            invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                            fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')');
+                        } catch (InvocationTargetException e) {
+                            assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                        }
+                        loop.checkError();
+                    }
+                    assertEquals("foreground retries must not burn the orphan attempt budget",
+                            0, readInt(loop, "catchUpCapGapAttempts"));
+                    assertEquals("foreground retries must not anchor an orphan cap-gap episode",
+                            -1L, readLong(loop, "catchUpCapGapFirstNanos"));
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() throws Exception {
+        // The strike count alone must NOT latch a terminal: escalation also requires the
+        // cap-gap episode to have persisted for catchUpCapGapMinEscalationWindowMillis.
+        //
+        // This keeps a routine rolling restart from quarantining a drainable orphan slot.
+        // MAX_CATCHUP_CAP_GAP_ATTEMPTS strikes accrue in ~2 minutes at the capped
+        // reconnect backoff -- less than the time the larger-cap node is away -- so a
+        // count-only budget would quarantine the slot on the very transient the budget
+        // exists to ride out. Here we drive far MORE than the budget's strikes inside a
+        // deliberately huge window and assert the orphan loop stays retriable.
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+            maxField.setAccessible(true);
+            int maxAttempts = maxField.getInt(null);
+            CatchUpCapturingClient client = new CatchUpCapturingClient(160);
+            try (CursorSendEngine engine = newEngine()) {
+                // A one-hour dwell the test cannot possibly elapse.
+                CursorWebSocketSendLoop loop = newLoop(engine, client, 3_600_000L);
+                try {
+                    seedMirror(loop, TestUtils.repeat("x", 200));
+                    for (int i = 1; i <= maxAttempts + 4; i++) {
+                        try {
+                            invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                            fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')');
+                        } catch (InvocationTargetException e) {
+                            assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                        }
+                        // The producer-facing latch must stay clear on EVERY attempt,
+                        // including the ones past the strike budget.
+                        loop.checkError();
+                    }
+                    assertTrue("the strikes really did exceed the budget",
+                            readInt(loop, "catchUpCapGapAttempts") > maxAttempts);
+
+                    // Backdate the episode anchor past the window: the very next cap gap
+                    // now satisfies BOTH conditions and latches. This pins the AND -- if
+                    // escalation ignored the wall clock the loop would already have
+                    // latched above; if it ignored the strike count it could never latch.
+                    Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos");
+                    anchor.setAccessible(true);
+                    anchor.setLong(loop, System.nanoTime() - TimeUnit.HOURS.toNanos(2));
+                    try {
+                        invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                        fail("the escalating cap gap must still raise CatchUpSendException");
+                    } catch (InvocationTargetException e) {
+                        assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                    }
+                    try {
+                        loop.checkError();
+                        fail("a cap gap that outlives the escalation window must latch a terminal");
+                    } catch (LineSenderException terminal) {
+                        assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(),
+                                terminal.getMessage().contains("during catch-up")
+                                        && terminal.getMessage().contains("must be resent"));
+                    }
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testCatchUpCapGapDwellConversionSaturatesInsteadOfOverflowing() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            long maxExactMillis = Long.MAX_VALUE / 1_000_000L;
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop exactLoop = newLoop(
+                        engine, new CatchUpCapturingClient(0), maxExactMillis);
+                try {
+                    assertEquals(maxExactMillis * 1_000_000L,
+                            readLong(exactLoop, "catchUpCapGapMinEscalationWindowNanos"));
+                } finally {
+                    exactLoop.close();
+                }
+
+                CursorWebSocketSendLoop saturatedLoop = newLoop(
+                        engine, new CatchUpCapturingClient(0), maxExactMillis + 1L);
+                try {
+                    assertEquals("an oversized dwell must become effectively infinite, not negative",
+                            Long.MAX_VALUE,
+                            readLong(saturatedLoop, "catchUpCapGapMinEscalationWindowNanos"));
+                } finally {
+                    saturatedLoop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testCapGapEpisodeWithANegativeAnchorStillEscalates() throws Exception {
+        // A cap-gap episode anchored at a NEGATIVE nanoTime instant must escalate like any
+        // other. A System.nanoTime() value is only meaningful as a difference -- its origin
+        // is arbitrary and the spec permits negative values -- so no state may ride on the
+        // anchor's sign. sendDictCatchUp once tested catchUpCapGapFirstNanos < 0 to mean "no
+        // episode open": that read a negative anchor as unset, re-anchored it to now on every
+        // strike and pinned episodeNanos at ~0, so the dwell was never satisfied and the
+        // terminal could never latch, however long the cap gap truly persisted.
+        //
+        // That is what reddened CI. The sibling test above backdates the anchor two hours,
+        // and on Linux nanoTime() is nanos-since-boot: on a CI agent up ten minutes it is
+        // ~6e11, so "two hours ago" comes out ~ -6.6e12 -- negative. The defect therefore
+        // only surfaced where uptime is under that backdate: every fresh CI agent, and never
+        // a long-lived dev box (which is why it passed locally). Planting the negative anchor
+        // directly pins the sentinel on ANY machine, whatever its uptime.
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+            maxField.setAccessible(true);
+            int maxAttempts = maxField.getInt(null);
+            CatchUpCapturingClient client = new CatchUpCapturingClient(160);
+            try (CursorSendEngine engine = newEngine()) {
+                // A one-hour dwell, against an anchor two hours back: satisfied on elapsed,
+                // but only if the negative anchor survives to the subtraction.
+                CursorWebSocketSendLoop loop = newLoop(engine, client, 3_600_000L);
+                try {
+                    seedMirror(loop, TestUtils.repeat("x", 200));
+                    // Satisfy the strike half of the AND, one short of the budget.
+                    for (int i = 1; i < maxAttempts; i++) {
+                        try {
+                            invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                            fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')');
+                        } catch (InvocationTargetException e) {
+                            assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                        }
+                        loop.checkError(); // dwell unmet => retriable, whatever the count
+                    }
+                    // The episode began two hours ago on a machine booted minutes ago.
+                    Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos");
+                    anchor.setAccessible(true);
+                    anchor.setLong(loop, -TimeUnit.HOURS.toNanos(2));
+
+                    // Both halves now hold, so this strike must latch the terminal.
+                    try {
+                        invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                        fail("the escalating cap gap must still raise CatchUpSendException");
+                    } catch (InvocationTargetException e) {
+                        assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                    }
+                    try {
+                        loop.checkError();
+                        fail("a cap-gap episode anchored at a negative nanoTime instant must still "
+                                + "escalate -- the anchor's sign carries no meaning");
+                    } catch (LineSenderException terminal) {
+                        assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(),
+                                terminal.getMessage().contains("during catch-up")
+                                        && terminal.getMessage().contains("must be resent"));
+                    }
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Exception {
+        // A TRANSIENT catch-up failure (the wire drops mid-catch-up -- a flapping LB, a
+        // reset) must never increment the cap-gap terminal budget. The budget exists to
+        // prove a PERSISTENT cluster capability gap; letting a transient feed it means
+        // enough wire flaps hard-fail a live store-and-forward producer, which is the
+        // exact failure store-and-forward promises cannot happen.
+        //
+        // The production code is correct, but nothing pinned it: the counter is never
+        // read by the existing transient test, and one transient can never reach a
+        // 16-strike budget anyway. So drive MORE transients than the whole budget and
+        // assert the counter never moves and no terminal ever latches.
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+            maxField.setAccessible(true);
+            int maxAttempts = maxField.getInt(null);
+            // A cap that FITS (no cap gap), but whose sendBinary always throws: every
+            // failure here is transport-transient, never a capability gap.
+            CatchUpCapturingClient client = new CatchUpCapturingClient(0, true);
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newLoop(engine, client, 0L);
+                try {
+                    seedMirror(loop, "alpha");
+                    for (int i = 1; i <= maxAttempts + 4; i++) {
+                        try {
+                            invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                            fail("a transient send failure must raise CatchUpSendException (attempt " + i + ')');
+                        } catch (InvocationTargetException e) {
+                            assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                        }
+                        loop.checkError(); // a transient is retriable, forever
+                        assertEquals("a transient must NOT burn the cap-gap terminal budget",
+                                0, readInt(loop, "catchUpCapGapAttempts"));
+                        assertEquals("a transient must NOT anchor a cap-gap episode",
+                                -1L, readLong(loop, "catchUpCapGapFirstNanos"));
+                    }
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testTransportOutageRestartsCapGapEpisode() throws Exception {
+        assertUnrelatedReconnectStateRestartsCapGapEpisode(false);
+    }
+
+    @Test
+    public void testRoleRejectRestartsCapGapEpisode() throws Exception {
+        assertUnrelatedReconnectStateRestartsCapGapEpisode(true);
+    }
+
+    @Test
+    public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception {
+        // M1: an entry too large for the fresh server's cap during catch-up (a
+        // heterogeneous / rolling-cap failover to a smaller-cap node) must NOT latch
+        // on first sight. sendDictCatchUp throws a RETRIABLE CatchUpSendException so
+        // the reconnect loop rides it out -- a larger-cap node may return -- and only
+        // after MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive cap gaps does it recordFatal.
+        // Pre-fix the first cap gap latched a terminal, so one transient failover to a
+        // smaller-cap node quarantined the orphan slot. (A successful catch-up resets the budget;
+        // the other catch-up tests, which use a fitting cap, never trip it.)
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+            maxField.setAccessible(true);
+            int maxAttempts = maxField.getInt(null);
+            // Pin the budget against a LITERAL before deriving anything from it. The
+            // retriable loop below is bounded by maxAttempts, so keying this test purely
+            // off the constant under test makes it TAUTOLOGICAL: a regression of
+            // MAX_CATCHUP_CAP_GAP_ATTEMPTS to 1 -- which is precisely the pre-fix bug this
+            // test names, a single cap gap quarantining the slot -- would run the loop ZERO
+            // times, the "exhausting" attempt would become the FIRST attempt, and the test
+            // would still pass green. Requiring > 1 makes that regression fail here, and it
+            // also guarantees the loop runs at least once, so the first cap gap is genuinely
+            // asserted retriable rather than vacuously skipped.
+            assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap, else a single "
+                            + "transient failover to a smaller-cap node quarantines the slot "
+                            + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']',
+                    maxAttempts > 1);
+            // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol.
+            CatchUpCapturingClient client = new CatchUpCapturingClient(160);
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    seedMirror(loop, TestUtils.repeat("x", 200));
+                    // Attempts 1 .. max-1 are retriable: no terminal is latched.
+                    for (int i = 1; i < maxAttempts; i++) {
+                        try {
+                            invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                            fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')');
+                        } catch (InvocationTargetException e) {
+                            assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                            assertTrue("attempt " + i + " must name the catch-up cap gap: "
+                                            + e.getCause().getMessage(),
+                                    e.getCause().getMessage().contains("during catch-up"));
+                        }
+                        loop.checkError(); // under budget => retriable => no terminal
+                    }
+                    // The exhausting attempt still throws, and now latches the terminal.
+                    try {
+                        invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                        fail("the exhausting cap gap must still raise CatchUpSendException");
+                    } catch (InvocationTargetException e) {
+                        assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                    }
+                    try {
+                        loop.checkError();
+                        fail("exhausting the cap-gap settle budget must latch a terminal");
+                    } catch (LineSenderException terminal) {
+                        assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(),
+                                terminal.getMessage().contains("during catch-up")
+                                        && terminal.getMessage().contains("must be resent"));
+                    }
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception {
+        // The cap-gap settle budget (catchUpCapGapAttempts) counts CONSECUTIVE cap
+        // gaps across reconnects; a successful catch-up ends the episode and MUST reset
+        // it to 0 (sendDictCatchUp's final line). Otherwise cap gaps interspersed with
+        // successful catch-ups -- a rolling-cap cluster where a larger-cap node comes
+        // and goes -- would accumulate to a spurious terminal over a long-lived orphan drainer.
+        // testCatchUpCapGapRetriesUntilBudgetThenLatches only accrues gaps under one
+        // fixed cap with no success interleaved, so it cannot pin the reset.
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+            maxField.setAccessible(true);
+            int maxAttempts = maxField.getInt(null);
+            // Same anti-tautology pin as testCatchUpCapGapRetriesUntilBudgetThenLatches.
+            // With maxAttempts == 1 the accrual loop below would run ZERO times and the
+            // "budget accrued to max-1" precondition would degenerate to 0 == 0, so the
+            // reset-to-0 assertion that is the whole point of this test would prove nothing.
+            assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap "
+                            + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']',
+                    maxAttempts > 1);
+            CatchUpCapturingClient client = new CatchUpCapturingClient(160); // too small for a 200-char symbol
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    seedMirror(loop, TestUtils.repeat("x", 200));
+                    // Accrue max-1 consecutive cap gaps (each retriable, no terminal).
+                    for (int i = 1; i < maxAttempts; i++) {
+                        try {
+                            invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                            fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')');
+                        } catch (InvocationTargetException e) {
+                            assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                        }
+                    }
+                    assertEquals("precondition: budget accrued to max-1",
+                            maxAttempts - 1, readInt(loop, "catchUpCapGapAttempts"));
+
+                    // A larger-cap node returns: the whole dictionary re-registers with
+                    // no cap gap, so the settle budget must reset to 0.
+                    client.cap = 0; // no cap => the 200-char symbol fits one frame
+                    invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                    assertEquals("a successful catch-up must reset the cap-gap settle budget",
+                            0, readInt(loop, "catchUpCapGapAttempts"));
+
+                    // Behavioural proof the budget is genuinely fresh: max-1 more cap
+                    // gaps still latch NO terminal (they would if the counter had stayed
+                    // at max-1 -- one more gap would have quarantined the slot).
+                    client.cap = 160;
+                    for (int i = 1; i < maxAttempts; i++) {
+                        try {
+                            invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+                            fail("post-reset cap gap must be retriable (attempt " + i + ')');
+                        } catch (InvocationTargetException e) {
+                            assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+                        }
+                        loop.checkError(); // fresh budget => still under max => no terminal
+                    }
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testMirrorOverflowFailsLoud() throws Exception {
+        // ensureSentDictCapacity must latch a terminal -- not silently overflow the
+        // int capacity math into a heap-corrupting copyMemory -- when the sent-dict
+        // mirror would exceed MAX_SENT_DICT_BYTES. Unreachable at real cardinality
+        // (~200M+ symbols on one connection), so drive the guard directly with an
+        // oversized required, mirroring testCatchUpChunkFrameSizeOverflowFailsLoud.
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_SENT_DICT_BYTES");
+            maxField.setAccessible(true);
+            long overCeiling = (long) maxField.getInt(null) + 1L;
+            CatchUpCapturingClient client = new CatchUpCapturingClient(0);
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("ensureSentDictCapacity", long.class);
+                    m.setAccessible(true);
+                    try {
+                        m.invoke(loop, overCeiling);
+                        fail("a mirror capacity past MAX_SENT_DICT_BYTES must fail loud, not overflow");
+                    } catch (InvocationTargetException e) {
+                        assertEquals("overflow must surface as LineSenderException",
+                                "LineSenderException", e.getCause().getClass().getSimpleName());
+                        assertTrue("message must name the mirror ceiling: " + e.getCause().getMessage(),
+                                e.getCause().getMessage().contains("mirror exceeds the maximum size"));
+                    }
+                    // recordFatal (not a bare throw) latched the terminal, so the loop
+                    // winds down instead of reconnecting into the same overflow.
+                    try {
+                        loop.checkError();
+                        fail("mirror overflow must latch a terminal");
+                    } catch (LineSenderException terminal) {
+                        assertTrue(terminal.getMessage().contains("mirror exceeds the maximum size"));
+                    }
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    private static void appendFrames(CursorSendEngine engine, int count) {
+        long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+        try {
+            byte[] payload = "frame-bytes-padd".getBytes(StandardCharsets.US_ASCII);
+            for (int i = 0; i < payload.length; i++) {
+                Unsafe.getUnsafe().putByte(buf + i, payload[i]);
+            }
+            for (int i = 0; i < count; i++) {
+                engine.appendBlocking(buf, 16);
+            }
+        } finally {
+            Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
+    private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReject) throws Exception {
+        // Accrue an orphan drainer's cap-gap strikes to one short of terminal, then
+        // simulate a long unrelated outage before another small-cap node appears. The
+        // outage must end the old episode: its wall-clock duration says nothing about
+        // whether the cluster's batch cap remained incompatible while no node answered.
+        TestUtils.assertMemoryLeak(() -> {
+            Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+            maxField.setAccessible(true);
+            int maxAttempts = maxField.getInt(null);
+            assertTrue("the cap-gap settle budget must have a retriable interval", maxAttempts > 1);
+
+            int[] reconnectCalls = {0};
+            long[] staleAnchor = {Long.MIN_VALUE};
+            CursorWebSocketSendLoop[] loopRef = new CursorWebSocketSendLoop[1];
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+                        null, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+                        () -> {
+                            int call = ++reconnectCalls[0];
+                            if (call < maxAttempts) {
+                                return new CatchUpCapturingClient(160);
+                            }
+                            if (call == maxAttempts) {
+                                assertEquals("precondition: consecutive cap gaps survive reconnect",
+                                        maxAttempts - 1,
+                                        readInt(loopRef[0], "catchUpCapGapAttempts"));
+                                // Model the elapsed outage without sleeping. With the defect,
+                                // this old anchor survives the unrelated failure and the next
+                                // cap gap immediately satisfies both terminal conditions.
+                                staleAnchor[0] = System.nanoTime() - TimeUnit.HOURS.toNanos(2);
+                                setField(loopRef[0], "catchUpCapGapFirstNanos", staleAnchor[0]);
+                                if (roleReject) {
+                                    throw new QwpRoleMismatchException(
+                                            "PRIMARY", null, "all endpoints role-rejected");
+                                }
+                                throw new LineSenderException("transport unavailable");
+                            }
+                            if (call == maxAttempts + 1) {
+                                // Stop after getServerMaxBatchSize() has driven the final cap
+                                // gap, leaving its fresh episode state observable below.
+                                return new CatchUpCapturingClient(160, false,
+                                        () -> setBooleanFieldUnchecked(loopRef[0], "running", false));
+                            }
+                            throw new AssertionError("unexpected reconnect call " + call);
+                        },
+                        5_000L, 0L, 0L, false,
+                        CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
+                        CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
+                        0L, TimeUnit.HOURS.toMillis(1),
+                        CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET);
+                loopRef[0] = loop;
+                try {
+                    seedMirror(loop, TestUtils.repeat("x", 200));
+                    setBooleanField(loop, "running", true);
+                    invokeConnectLoop(loop);
+
+                    loop.checkError();
+                    assertEquals("pre-outage cap gaps must not carry into the new episode",
+                            1, readInt(loop, "catchUpCapGapAttempts"));
+                    assertTrue("the post-outage cap gap must get a fresh dwell anchor",
+                            readLong(loop, "catchUpCapGapFirstNanos") > staleAnchor[0]);
+                    assertEquals("test must observe gaps, the unrelated state, and a new gap",
+                            maxAttempts + 1, reconnectCalls[0]);
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    // Builds a QWP delta frame [12-byte header][deltaStart varint][deltaCount
+    // varint][ [len varint][utf8] ... ] for the given symbols. accumulateSentDict
+    // skips the header, so its content is irrelevant; the caller frees the frame.
+    private static long buildDeltaFrame(int deltaStart, String[] symbols, int[] outLen) {
+        int deltaCount = symbols.length;
+        int size = 12 + varintSize(deltaStart) + varintSize(deltaCount);
+        for (String s : symbols) {
+            size += varintSize(s.getBytes(StandardCharsets.UTF_8).length)
+                    + s.getBytes(StandardCharsets.UTF_8).length;
+        }
+        long addr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT);
+        long p = writeVarint(addr + 12, deltaStart);
+        p = writeVarint(p, deltaCount);
+        for (String s : symbols) {
+            byte[] b = s.getBytes(StandardCharsets.UTF_8);
+            p = writeVarint(p, b.length);
+            for (byte x : b) {
+                Unsafe.getUnsafe().putByte(p++, x);
+            }
+        }
+        outLen[0] = size;
+        return addr;
+    }
+
+    private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getInt(loop);
+    }
+
+    // Parses the loop's native sent-dictionary mirror ([len varint][utf8]...) back
+    // into the symbol strings a reconnect catch-up would re-register.
+    private static List readMirrorSymbols(CursorWebSocketSendLoop loop) throws Exception {
+        long addr = readLong(loop, "sentDictBytesAddr");
+        int len = readInt(loop, "sentDictBytesLen");
+        List out = new ArrayList<>();
+        long p = addr;
+        long limit = addr + len;
+        while (p < limit) {
+            long l = 0;
+            int shift = 0;
+            while (p < limit) {
+                byte b = Unsafe.getUnsafe().getByte(p++);
+                l |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    break;
+                }
+                shift += 7;
+            }
+            byte[] bytes = new byte[(int) l];
+            for (int i = 0; i < l; i++) {
+                bytes[i] = Unsafe.getUnsafe().getByte(p++);
+            }
+            out.add(new String(bytes, StandardCharsets.UTF_8));
+        }
+        return out;
+    }
+
+    // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response
+    // handler, mimicking the server acking a catch-up frame (which carries no tables).
+    private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception {
+        int size = 11; // status(1) + sequence(8) + tableCount(2)
+        long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT);
+        try {
+            Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK);
+            Unsafe.getUnsafe().putLong(ptr + 1, wireSeq);
+            Unsafe.getUnsafe().putShort(ptr + 9, (short) 0);
+            Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler");
+            f.setAccessible(true);
+            Object handler = f.get(loop);
+            Method m = handler.getClass().getDeclaredMethod("onBinaryMessage", long.class, int.class);
+            m.setAccessible(true);
+            m.invoke(handler, ptr, size);
+        } finally {
+            Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
+    private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loop, long replayStart) throws Exception {
+        Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("setWireBaselineWithCatchUp", long.class);
+        m.setAccessible(true);
+        m.invoke(loop, replayStart);
+    }
+
+    private static void invokeConnectLoop(CursorWebSocketSendLoop loop) throws Exception {
+        Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(
+                "connectLoop", Throwable.class, String.class, long.class);
+        m.setAccessible(true);
+        m.invoke(loop, new LineSenderException("test reconnect"), "reconnect", 0L);
+    }
+
+    private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) {
+        return newLoop(engine, client, 0L);
+    }
+
+    /**
+     * As {@link #newLoop(CursorSendEngine, WebSocketClient)} but with an explicit
+     * cap-gap escalation dwell. These white-box tests model an orphan drainer, where
+     * {@code 0} means count-only quarantine; foreground loops retry indefinitely.
+     */
+    private CursorWebSocketSendLoop newLoop(
+            CursorSendEngine engine, WebSocketClient client, long capGapWindowMillis
+    ) {
+        return new CursorWebSocketSendLoop(
+                client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+                () -> {
+                    throw new UnsupportedOperationException("test loop is never started");
+                },
+                5_000L, 100L, 5_000L, false,
+                CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
+                CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
+                0L, capGapWindowMillis,
+                CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET);
+    }
+
+    private CursorWebSocketSendLoop newForegroundLoop(
+            CursorSendEngine engine, WebSocketClient client
+    ) {
+        // Deliberately use the compatibility overload: its safe default must remain the
+        // foreground RETRY_FOREVER policy for external callers.
+        return new CursorWebSocketSendLoop(
+                client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+                () -> {
+                    throw new UnsupportedOperationException("test loop is never started");
+                },
+                5_000L, 100L, 5_000L, false,
+                CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
+                CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
+                0L, 0L);
+    }
+
+    private CursorSendEngine newEngine() {
+        return new CursorSendEngine(tmpDir, 16_384);
+    }
+
+    private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getLong(loop);
+    }
+
+    // Populates the loop's native sent-dictionary mirror with {@code symbols} in
+    // the on-wire [len varint][utf8] layout, so setWireBaselineWithCatchUp sees a
+    // non-empty dictionary to re-register. loop.close() frees it.
+    private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) throws Exception {
+        int total = 0;
+        for (String s : symbols) {
+            int len = s.getBytes(StandardCharsets.UTF_8).length;
+            total += varintSize(len) + len;
+        }
+        long addr = Unsafe.malloc(total, MemoryTag.NATIVE_DEFAULT);
+        long p = addr;
+        for (String s : symbols) {
+            byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+            p = writeVarint(p, bytes.length);
+            for (byte b : bytes) {
+                Unsafe.getUnsafe().putByte(p++, b);
+            }
+        }
+        setField(loop, "sentDictBytesAddr", addr);
+        setBooleanField(loop, "sentDictBytesOwned", true);
+        setIntField(loop, "sentDictBytesCapacity", total);
+        setIntField(loop, "sentDictBytesLen", total);
+        setIntField(loop, "sentDictCount", symbols.length);
+    }
+
+    private static void setField(Object target, String name, long value) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        f.setLong(target, value);
+    }
+
+    private static void setIntField(Object target, String name, int value) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        f.setInt(target, value);
+    }
+
+    private static void setBooleanField(Object target, String name, boolean value) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        f.setBoolean(target, value);
+    }
+
+    private static void setBooleanFieldUnchecked(Object target, String name, boolean value) {
+        try {
+            setBooleanField(target, name, value);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    private static int varintSize(long value) {
+        int n = 1;
+        while (value > 0x7F) {
+            value >>>= 7;
+            n++;
+        }
+        return n;
+    }
+
+    private static long writeVarint(long addr, long value) {
+        while (value > 0x7F) {
+            Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80));
+            value >>>= 7;
+        }
+        Unsafe.getUnsafe().putByte(addr++, (byte) value);
+        return addr;
+    }
+
+    // Stub transport: completes no real I/O. getServerMaxBatchSize drives the
+    // catch-up split; sendBinary counts the frames the catch-up emitted, or --
+    // when throwOnSend is set -- raises a transient wire error to model the fresh
+    // connection dropping mid-catch-up.
+    private static final class CatchUpCapturingClient extends WebSocketClient {
+        // Mutable so a test can model a rolling-cap cluster: raise it for a node that
+        // accepts the dictionary, lower it for a smaller-cap node that cap-gaps.
+        private int cap;
+        private final Runnable onCapRead;
+        private final boolean throwOnSend;
+        private int framesSent;
+
+        CatchUpCapturingClient(int cap) {
+            this(cap, false);
+        }
+
+        CatchUpCapturingClient(int cap, boolean throwOnSend) {
+            this(cap, throwOnSend, null);
+        }
+
+        CatchUpCapturingClient(int cap, boolean throwOnSend, Runnable onCapRead) {
+            super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+            this.cap = cap;
+            this.throwOnSend = throwOnSend;
+            this.onCapRead = onCapRead;
+        }
+
+        @Override
+        public int getServerMaxBatchSize() {
+            if (onCapRead != null) {
+                onCapRead.run();
+            }
+            return cap;
+        }
+
+        @Override
+        public int getServerQwpVersion() {
+            return 1;
+        }
+
+        @Override
+        public void sendBinary(long dataPtr, int length) {
+            if (throwOnSend) {
+                throw new RuntimeException("transient wire failure during catch-up");
+            }
+            framesSent++;
+        }
+
+        @Override
+        protected void ioWait(int timeout, int op) {
+        }
+
+        @Override
+        protected void setupIoWait() {
+        }
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java
new file mode 100644
index 00000000..064edf3f
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java
@@ -0,0 +1,254 @@
+/*+*****************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.http.client.WebSocketClientFactory;
+import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException;
+import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.IdentityHashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class CursorWebSocketSendLoopForegroundReconnectPolicyTest {
+    private static final long SEGMENT_SIZE_BYTES = 16_384L;
+
+    @Rule
+    public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build();
+
+    @Test
+    public void testPostStartAuthFailureRetriesUntilCredentialsRecover() throws Exception {
+        assertForegroundRecovers(false,
+                () -> new QwpAuthFailedException(401, "localhost", 1));
+    }
+
+    @Test
+    public void testPostStartDurableAckMismatchRetriesUntilCapabilityRecovers() throws Exception {
+        assertForegroundRecovers(true,
+                () -> new QwpDurableAckMismatchException("localhost", 1, "primary"));
+    }
+
+    private void assertForegroundRecovers(boolean durableAck, FailureSupplier failureSupplier) throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck);
+            try (TestWebSocketServer server = new TestWebSocketServer(handler, true);
+                 CursorSendEngine engine = new CursorSendEngine(
+                         sfDir.newFolder().getAbsolutePath(), SEGMENT_SIZE_BYTES)) {
+                server.start();
+                Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+                WebSocketClient initialClient = connect(server.getPort(), durableAck);
+                ScriptedFactory factory = new ScriptedFactory(
+                        server.getPort(), durableAck, failureSupplier);
+                CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+                        initialClient,
+                        engine,
+                        0L,
+                        CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+                        factory,
+                        100L,
+                        1L,
+                        4L,
+                        durableAck,
+                        durableAck ? 10L : 0L,
+                        CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
+                        0L,
+                        0L,
+                        CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
+                try {
+                    appendFrame(engine, (byte) 1);
+                    loop.start();
+
+                    await(() -> loop.getTotalReconnects() >= 1L
+                                    || loop.getTerminalError() != null,
+                            "foreground reconnect did not complete");
+                    Assert.assertNull("post-start endpoint-policy failures must not stop the producer",
+                            loop.getTerminalError());
+                    Assert.assertTrue("the reconnect loop must retry both scripted failures",
+                            factory.attempts() >= 3);
+
+                    long target = appendFrame(engine, (byte) 2);
+                    await(() -> engine.ackedFsn() >= target || loop.getTerminalError() != null,
+                            "foreground sender did not deliver after the policy failure cleared");
+                    Assert.assertNull(loop.getTerminalError());
+                    Assert.assertEquals(target, engine.ackedFsn());
+                } finally {
+                    loop.close();
+                    initialClient.close();
+                }
+            }
+        });
+    }
+
+    private static long appendFrame(CursorSendEngine engine, byte marker) {
+        long buffer = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+        try {
+            for (int i = 0; i < 16; i++) {
+                Unsafe.getUnsafe().putByte(buffer + i, marker);
+            }
+            return engine.appendBlocking(buffer, 16);
+        } finally {
+            Unsafe.free(buffer, 16, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
+    private static void await(Condition condition, String failureMessage) throws Exception {
+        long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+        while (!condition.isTrue()) {
+            if (System.nanoTime() >= deadline) {
+                Assert.fail(failureMessage);
+            }
+            Thread.sleep(1L);
+        }
+    }
+
+    private static WebSocketClient connect(int port, boolean durableAck) throws Exception {
+        WebSocketClient client = WebSocketClientFactory.newPlainTextInstance();
+        try {
+            client.setQwpMaxVersion(1);
+            client.setQwpRequestDurableAck(durableAck);
+            client.setConnectTimeout(5_000);
+            client.connect("localhost", port);
+            client.upgrade("/write/v4", 5_000, null);
+            return client;
+        } catch (Throwable e) {
+            client.close();
+            throw e;
+        }
+    }
+
+    @FunctionalInterface
+    private interface Condition {
+        boolean isTrue() throws Exception;
+    }
+
+    @FunctionalInterface
+    private interface FailureSupplier {
+        Exception get();
+    }
+
+    private static final class ScriptedFactory implements CursorWebSocketSendLoop.ReconnectFactory {
+        private final AtomicInteger attempts = new AtomicInteger();
+        private final boolean durableAck;
+        private final FailureSupplier failureSupplier;
+        private final int port;
+
+        private ScriptedFactory(int port, boolean durableAck, FailureSupplier failureSupplier) {
+            this.port = port;
+            this.durableAck = durableAck;
+            this.failureSupplier = failureSupplier;
+        }
+
+        int attempts() {
+            return attempts.get();
+        }
+
+        @Override
+        public WebSocketClient reconnect() throws Exception {
+            if (attempts.incrementAndGet() <= 2) {
+                throw failureSupplier.get();
+            }
+            return connect(port, durableAck);
+        }
+    }
+
+    private static final class DropFirstConnectionHandler
+            implements TestWebSocketServer.WebSocketServerHandler {
+        private static final String TABLE = "trades";
+        private final boolean durableAck;
+        private final Map wireSeqByConnection =
+                new IdentityHashMap<>();
+        private TestWebSocketServer.ClientHandler firstConnection;
+
+        private DropFirstConnectionHandler(boolean durableAck) {
+            this.durableAck = durableAck;
+        }
+
+        @Override
+        public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            long[] sequence = wireSeqByConnection.get(client);
+            if (sequence == null) {
+                sequence = new long[1];
+                wireSeqByConnection.put(client, sequence);
+                if (firstConnection == null) {
+                    firstConnection = client;
+                }
+            }
+            long wireSeq = sequence[0]++;
+            try {
+                client.sendBinary(okFrame(wireSeq));
+                if (durableAck) {
+                    client.sendBinary(durableAckFrame(wireSeq));
+                }
+                if (client == firstConnection) {
+                    client.close();
+                }
+            } catch (IOException ignored) {
+                // The deliberate close may race the acknowledgement write.
+            }
+        }
+
+        private static byte[] durableAckFrame(long seqTxn) {
+            byte[] name = TABLE.getBytes(StandardCharsets.UTF_8);
+            ByteBuffer bb = ByteBuffer.allocate(1 + 2 + 2 + name.length + 8)
+                    .order(ByteOrder.LITTLE_ENDIAN);
+            bb.put((byte) 0x02);
+            bb.putShort((short) 1);
+            bb.putShort((short) name.length);
+            bb.put(name);
+            bb.putLong(seqTxn);
+            return bb.array();
+        }
+
+        private static byte[] okFrame(long wireSeq) {
+            byte[] name = TABLE.getBytes(StandardCharsets.UTF_8);
+            ByteBuffer bb = ByteBuffer.allocate(1 + 8 + 2 + 2 + name.length + 8)
+                    .order(ByteOrder.LITTLE_ENDIAN);
+            bb.put((byte) 0x00);
+            bb.putLong(wireSeq);
+            bb.putShort((short) 1);
+            bb.putShort((short) name.length);
+            bb.put(name);
+            bb.putLong(wireSeq);
+            return bb.array();
+        }
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
new file mode 100644
index 00000000..116cbc55
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
@@ -0,0 +1,328 @@
+/*+*****************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Guards the recovery-seeded symbol-dictionary mirror against leaking when the
+ * I/O loop is constructed but never run.
+ * 

+ * On recovery / orphan-drain the {@link CursorWebSocketSendLoop} constructor
+ * seeds a native mirror ({@code sentDictBytesAddr}) from the slot's persisted
+ * dictionary so the first connection can re-register it. That mirror is freed on
+ * the I/O thread's exit path -- so if the loop is closed WITHOUT ever starting
+ * (start() never called, or Thread.start() failing before the loop runs), the
+ * free never happens. {@code close()} must free it in that case.
+ */
+public class CursorWebSocketSendLoopMirrorLeakTest {
+
+    private static final int DISTINCT_SYMBOLS = 8;
+    private static final int ROWS = 40;
+
+    @Test
+    public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception {
+        Path sfDir = Files.createTempDirectory("qwp-mirror-leak");
+        try {
+            // Populate a slot with delta frames + a non-empty .symbol-dict, then
+            // abandon it (silent server, close-fast) -- outside assertMemoryLeak,
+            // because a full Sender+server round trip is not net-zero on its own.
+            populateRecoverableSlot(sfDir);
+
+            Path slot = sfDir.resolve("default");
+            Assert.assertTrue("populate must leave a persisted dictionary",
+                    Files.exists(slot.resolve(".symbol-dict")));
+
+            // Only the recovery construct + close is leak-checked: the engine
+            // recovers (loading the dict), the loop ctor seeds the mirror from it,
+            // and close() -- with NO start() -- must free every native allocation.
+            // Pre-fix the seeded mirror leaks here and this assertion fails.
+            assertMemoryLeak(() -> {
+                try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) {
+                    PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                    Assert.assertNotNull("disk-mode engine must open a persisted dict", pd);
+                    Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)",
+                            pd.size() > 0 && pd.loadedEntriesLen() > 0);
+                    long persistedAddr = pd.loadedEntriesAddr();
+
+                    CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+                            null, engine, 0, 1_000_000L,
+                            () -> {
+                                throw new IOException("no reconnect in this test");
+                            },
+                            0, 0, 1);
+                    // Close without start(): the ctor-seeded mirror is this
+                    // thread's to free, since the I/O loop never ran.
+                    Assert.assertTrue("precondition: the ctor seeded a non-empty mirror",
+                            readInt(loop, "sentDictCount") > 0);
+                    Assert.assertEquals("foreground loop must take the persisted buffer without copying",
+                            persistedAddr, readLong(loop, "sentDictBytesAddr"));
+                    Assert.assertEquals("ownership transfer must clear the persisted pointer",
+                            0L, pd.loadedEntriesAddr());
+                    Assert.assertTrue("foreground mirror must own the transferred buffer",
+                            readBoolean(loop, "sentDictBytesOwned"));
+                    loop.close();
+                    // close() must reset sentDictCount alongside freeing the buffer,
+                    // so the mirror stays all-or-nothing: a hypothetical post-close
+                    // start() (no closed guard) cannot read a stale count against a
+                    // freed buffer and drive a null-mirror catch-up.
+                    Assert.assertEquals("close() must reset sentDictCount to 0",
+                            0, readInt(loop, "sentDictCount"));
+                }
+            });
+        } finally {
+            rmDir(sfDir);
+        }
+    }
+
+    @Test
+    public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
+        // C1 regression: the orphan drainer (BackgroundDrainer) builds a NEW
+        // CursorWebSocketSendLoop per wire session against the SAME, persistent
+        // engine when a durable-ack capability gap forces a mid-drain recycle. The
+        // recovery mirror seed must survive that recycle. If the first loop consumes
+        // the persisted dictionary's loaded entries (a one-shot ownership transfer),
+        // the second loop seeds an EMPTY mirror (sentDictCount = 0), sends no
+        // reconnect catch-up, and the first replayed delta frame (deltaStart > 0)
+        // trips the torn-dict guard -- falsely quarantining a healthy slot with a
+        // bogus "resend required" terminal. Borrowing the entries leaves the
+        // dictionary intact for the engine's lifetime without making another native
+        // copy, so every recycled loop can re-seed.
+        Path sfDir = Files.createTempDirectory("qwp-mirror-reseed");
+        try {
+            populateRecoverableSlot(sfDir);
+            Path slot = sfDir.resolve("default");
+            assertMemoryLeak(() -> {
+                try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) {
+                    PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                    Assert.assertNotNull(pd);
+                    int dictSize = pd.size();
+                    Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0);
+                    long persistedAddr = pd.loadedEntriesAddr();
+
+                    // Session 1 seeds its mirror from the persisted dictionary.
+                    CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine);
+                    try {
+                        Assert.assertEquals("session-1 mirror must seed from the persisted dict",
+                                dictSize, readInt(loop1, "sentDictCount"));
+                        Assert.assertEquals("orphan session must borrow the persisted bytes",
+                                persistedAddr, readLong(loop1, "sentDictBytesAddr"));
+                        Assert.assertFalse("borrowed orphan mirror must not own the persisted bytes",
+                                readBoolean(loop1, "sentDictBytesOwned"));
+                    } finally {
+                        loop1.close();
+                    }
+                    Assert.assertEquals("closing a borrowed loop must leave the engine prefix alive",
+                            persistedAddr, pd.loadedEntriesAddr());
+
+                    // Session 2 against the SAME engine (the drainer recycle): the
+                    // seed must NOT have been consumed -- the mirror must re-seed to
+                    // the full dictionary so the reconnect catch-up is complete.
+                    CursorWebSocketSendLoop loop2 = newRecoveryLoop(engine);
+                    try {
+                        Assert.assertEquals("recycled session-2 mirror must re-seed from the "
+                                        + "persisted dict (pre-fix it was 0)",
+                                dictSize, readInt(loop2, "sentDictCount"));
+                        Assert.assertEquals(persistedAddr, readLong(loop2, "sentDictBytesAddr"));
+                    } finally {
+                        loop2.close();
+                    }
+                }
+            });
+        } finally {
+            rmDir(sfDir);
+        }
+    }
+
+    @Test
+    public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception {
+        // C1 regression: the constructor seeds the recovery mirror in TWO steps -- a
+        // malloc from the persisted dictionary's intact prefix, then an extension from
+        // the surviving frames' own deltas (appendSymbolToMirror). If that second step
+        // throws (a native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling), the throw
+        // leaves the constructor with the object unpublished, so neither
+        // ensureConnected's catch nor BackgroundDrainer's finally can ever close() it --
+        // and the already-malloc'd prefix mirror leaks. The constructor must free it on
+        // the throw. Pre-fix, the mirror leaks here and assertMemoryLeak fails.
+        Path sfDir = Files.createTempDirectory("qwp-mirror-ctor-throw");
+        try {
+            // A torn-dict SUBSET: three delta frames a@0,b@1,c@2 survive on disk, but the
+            // .symbol-dict is rewritten to hold only [a,b] (a host-crash tail tear). On
+            // recovery pd.size()==2 seeds (mallocs) the mirror, then the frame-seed
+            // rebuilds c@2 from the surviving frame -- the append the fault interrupts.
+            populateThreeFrameSlot(sfDir);
+            Path slot = sfDir.resolve("default");
+            try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slot.toString())) {
+                Assert.assertNotNull(torn);
+                torn.appendSymbol("a");
+                torn.appendSymbol("b");
+                Assert.assertEquals(2, torn.size());
+            }
+
+            assertMemoryLeak(() -> {
+                try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) {
+                    PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                    Assert.assertNotNull("recovery must open the torn subset dict", pd);
+                    Assert.assertEquals("prefix seed must malloc a 2-entry mirror", 2, pd.size());
+                    Assert.assertTrue("the frame-seed path must run (frames out-reach the dict)",
+                            engine.recoveredMaxSymbolDeltaStart() > 0L);
+
+                    setMirrorSeedFault(true);
+                    try {
+                        new CursorWebSocketSendLoop(
+                                null, engine, 0, 1_000_000L,
+                                () -> {
+                                    throw new IOException("no reconnect in this test");
+                                },
+                                0, 0, 1);
+                        Assert.fail("ctor must propagate the injected mirror-seed failure");
+                    } catch (LineSenderException expected) {
+                        Assert.assertTrue("unexpected message: " + expected.getMessage(),
+                                expected.getMessage().contains("simulated mirror seed allocation failure"));
+                    } finally {
+                        setMirrorSeedFault(false);
+                    }
+                    Assert.assertTrue("failed foreground construction must leave the persisted prefix owned",
+                            pd.loadedEntriesAddr() != 0L);
+                    // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor
+                    // malloc'd was freed on the throw -- pre-fix it leaks here.
+                }
+            });
+        } finally {
+            rmDir(sfDir);
+        }
+    }
+
+    // Constructs a recovery send loop but does NOT start it: the ctor seeds the
+    // catch-up mirror synchronously, which is all these tests observe. The
+    // reconnect factory is never invoked.
+    private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine) {
+        return new CursorWebSocketSendLoop(
+                null, engine, 0, 1_000_000L,
+                () -> {
+                    throw new IOException("no reconnect in this test");
+                },
+                0, 0, 1,
+                false, 0L, 3, 0L, 0L,
+                CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET);
+    }
+
+    private static void populateRecoverableSlot(Path sfDir) throws Exception {
+        try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+            int port = silent.getPort();
+            silent.start();
+            Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+            String cfg = "ws::addr=localhost:" + port
+                    + ";sf_dir=" + sfDir
+                    + ";sf_max_bytes=4096"
+                    + ";close_flush_timeout_millis=0;";
+            try (Sender s1 = Sender.fromConfig(cfg)) {
+                for (int i = 0; i < ROWS; i++) {
+                    s1.table("m")
+                            .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS))
+                            .longColumn("v", i)
+                            .atNow();
+                    s1.flush();
+                }
+            }
+        }
+    }
+
+    // Three delta frames a@0, b@1, c@2, nothing acked, so all three survive and
+    // replay from frame 0. Paired with a dictionary truncated to [a,b], this is a
+    // torn-dict SUBSET whose recovery drives the constructor's frame-seed path.
+    private static void populateThreeFrameSlot(Path sfDir) throws Exception {
+        try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+            int port = silent.getPort();
+            silent.start();
+            Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+            String cfg = "ws::addr=localhost:" + port
+                    + ";sf_dir=" + sfDir
+                    + ";close_flush_timeout_millis=0;";
+            try (Sender s = Sender.fromConfig(cfg)) {
+                s.table("m").symbol("s", "a").longColumn("v", 0).atNow();
+                s.flush();
+                s.table("m").symbol("s", "b").longColumn("v", 1).atNow();
+                s.flush();
+                s.table("m").symbol("s", "c").longColumn("v", 2).atNow();
+                s.flush();
+            }
+        }
+    }
+
+    private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getInt(loop);
+    }
+
+    private static boolean readBoolean(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getBoolean(loop);
+    }
+
+    private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getLong(loop);
+    }
+
+    // Toggles the loop's @TestOnly mirror-seed fault flag. Reflection because the
+    // flag is package-private in the production package (this test is in a sibling
+    // test package), the same non-reflective-path-unavailable reason readInt uses.
+    private static void setMirrorSeedFault(boolean value) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField("forceMirrorSeedFailureForTest");
+        f.setAccessible(true);
+        f.setBoolean(null, value);
+    }
+
+    private static void rmDir(Path dir) throws IOException {
+        TestUtils.removeTmpDirRec(dir == null ? null : dir.toString());
+    }
+
+    private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler {
+        @Override
+        public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            // never acks -- the sender leaves everything unacked in the slot
+        }
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
index ff710fe1..d7010310 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
@@ -27,12 +27,16 @@
 import io.questdb.client.DefaultHttpClientConfiguration;
 import io.questdb.client.cutlass.http.client.WebSocketClient;
 import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
 import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
 import io.questdb.client.network.PlainSocketFactory;
 import io.questdb.client.std.Files;
 import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
 import io.questdb.client.std.Unsafe;
 import io.questdb.client.test.tools.TestUtils;
 import org.junit.After;
@@ -44,6 +48,8 @@
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -75,7 +81,9 @@
 public class CursorWebSocketSendLoopOrphanTailTest {
 
     private static final int FLAG_DEFER_COMMIT = 0x01;
+    private static final int FLAG_DELTA_SYMBOL_DICT = 0x08;
     private static final int HEADER_OFFSET_FLAGS = 5;
+    private static final int HEADER_SIZE = 12;
     private static final int MAGIC_MESSAGE = 0x31505751; // "QWP1" little-endian
 
     private String tmpDir;
@@ -221,6 +229,256 @@ public void testSlowPathReplaysBelowTailThenRetiresAndRecyclesOnce() throws Exce
         });
     }
 
+    @Test
+    public void testRecoveredMaxSymbolIdSpansSealedSegments() throws Exception {
+        // The torn-dict detector must walk the SEALED segments too, not just the active
+        // one. Every other test in the suite fits its whole slot in a single active
+        // segment, so the sealed walk in SegmentRing.maxSymbolDeltaEnd was dead code as
+        // far as the tests were concerned: deleting it left them all green while
+        // recoveredMaxSymbolId silently collapsed to -1.
+        //
+        // -1 is not a benign wrong answer. seedGlobalDictionaryFromPersisted's guard is
+        // `recoveredMaxSymbolId >= pd.size()`, so a value that is too LOW never fires: a
+        // torn dictionary is trusted, the producer resumes seeded from the short
+        // dictionary, and it hands ids the surviving frames already define to different
+        // symbols -- the exact silent misattribution the whole feature exists to prevent.
+        //
+        // The shape that needs the sealed walk is the crash the guard was built for. In a
+        // plain multi-segment slot the ACTIVE segment holds the newest frames and thus the
+        // highest ids, so an active-only walk happens to get the right answer. It stops
+        // being right when the active segment contributes NOTHING to the committed range
+        // -- here, a producer that died mid-transaction, leaving an aborted deferred tail
+        // long enough to overflow into a fresh segment. maxSymbolDeltaEnd skips those
+        // frames (fsn > recoveredCommitBoundaryFsn), so the highest COMMITTED id is left
+        // sitting in a SEALED segment, reachable only through the sealed walk.
+        TestUtils.assertMemoryLeak(() -> {
+            // Small segments so the tail really does roll one. Each frame is
+            // FRAME_HEADER_SIZE + (QWP HEADER_SIZE + 2) = 22 bytes, against 256 - 24
+            // usable, so a segment holds ~10.
+            try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 256)) {
+                // Committed delta frames registering ids 0,1,2 -- the highest COMMITTED
+                // ids in the slot.
+                for (int i = 0; i < 3; i++) {
+                    appendDeltaFrame(engine, false, i, 1);
+                }
+                assertNull("the committed frames must still be in the ACTIVE segment here",
+                        engine.firstSealed());
+                // The aborted transaction: deferred frames with no covering commit. Keep
+                // appending until they overflow the segment, then a few more, so the
+                // committed frames end up SEALED and the active segment holds nothing but
+                // the tail. Ids stay small so the test's one-byte varint encoding holds.
+                int deferredId = 3;
+                for (int i = 0; i < 64 && engine.firstSealed() == null; i++) {
+                    appendDeltaFrame(engine, true, deferredId++, 1);
+                }
+                assertNotNull("the deferred tail must have rolled a segment", engine.firstSealed());
+                for (int i = 0; i < 3; i++) {
+                    appendDeltaFrame(engine, true, deferredId++, 1);
+                }
+            }
+            try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 256)) {
+                assertTrue(engine.wasRecoveredFromDisk());
+                assertNotNull("the recovered slot must have a sealed segment", engine.firstSealed());
+                assertEquals("the last commit-bearing frame is fsn 2",
+                        2L, engine.recoveredCommitBoundaryFsn());
+                // ids 0,1,2 were introduced by the committed frames, all of which now live
+                // in a SEALED segment. The active segment holds only the deferred tail,
+                // which the walk skips -- so an active-only walk returns 0 and this comes
+                // back -1.
+                assertEquals("the highest COMMITTED id lives in a SEALED segment; "
+                                + "maxSymbolDeltaEnd must walk the sealed segments to see it",
+                        2L, engine.recoveredMaxSymbolId());
+            }
+        });
+    }
+
+    @Test
+    public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception {
+        // recoveredMaxSymbolId must reflect only COMMITTED (transmitted) frames, not
+        // the aborted orphan-tail frames trySendOne retires without ever sending. A
+        // host crash that tears the persisted dictionary down to the committed ids
+        // while an orphan-tail frame introduced a HIGHER id must NOT over-reject the
+        // resume: the producer never reuses an orphan id on the wire (the tail retires
+        // first), so counting it would inflate recoveredMaxSymbolId and make
+        // seedGlobalDictionaryFromPersisted fail a fully-recoverable slot. The
+        // maxSymbolDeltaEnd walk is therefore bounded to recoveredCommitBoundaryFsn.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                // fsn 0: commit-bearing delta frame registering ids 0,1.
+                appendDeltaFrame(engine, false, 0, 2);
+                // fsn 1: DEFERRED delta frame registering id 2 -- the orphan tail.
+                appendDeltaFrame(engine, true, 2, 1);
+            }
+            try (CursorSendEngine engine = newEngine()) {
+                assertTrue(engine.wasRecoveredFromDisk());
+                assertEquals("last commit-bearing frame", 0L, engine.recoveredCommitBoundaryFsn());
+                assertEquals("orphan tail tip", 1L, engine.recoveredOrphanTipFsn());
+                // Only the committed frame's ids (0,1) count -> highest id 1. The
+                // orphan-tail frame's id 2 is excluded, so a resume whose recovered
+                // dictionary holds ids 0,1 (size 2) is NOT over-rejected.
+                assertEquals("orphan-tail id 2 must be excluded from recoveredMaxSymbolId",
+                        1L, engine.recoveredMaxSymbolId());
+            }
+        });
+    }
+
+    @Test
+    public void testZeroCountDeltaFrameAnchorsRecoveredMaxSymbolIdAtItsBaseline() throws Exception {
+        // A committed delta frame that introduces NO new symbol (deltaCount == 0 -- a
+        // commit frame, or one whose rows only re-use existing ids) still carries
+        // deltaStart == the producer's baseline at encode time, because beginMessage
+        // ALWAYS sets FLAG_DELTA_SYMBOL_DICT. maxSymbolDeltaEnd counts it as
+        // deltaStart + deltaCount == deltaStart (NOT 0), so recoveredMaxSymbolId ==
+        // deltaStart - 1 even though the frame introduces nothing.
+        //
+        // This is the mechanism behind the torn-dict guard's deliberate CONSERVATIVE
+        // over-strand (see seedGlobalDictionaryFromPersisted): if a host crash tears
+        // the persisted dictionary below such a frame's baseline while its
+        // symbol-introducing predecessors were already acked and trimmed, both the
+        // seed-time guard (recoveredMaxSymbolId >= pd.size()) and the drainer's replay
+        // guard (deltaStart > sentDictCount) fire and quarantine the slot -- fail-clean
+        // "resend required" -- even though the frame's rows may reference only ids the
+        // truncated dictionary still holds.
+        //
+        // Counting the zero-count frame's baseline is load-bearing SAFETY: a "fix" that
+        // skipped zero-count frames (returning 0 for them) would UNDER-strand and let a
+        // genuinely torn dictionary through, silently shifting the dense id map. This
+        // pins that a zero-count delta frame is anchored at its baseline, not skipped.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                // fsn 0: commit-bearing delta frame that genuinely registers ids 0..4.
+                appendDeltaFrame(engine, false, 0, 5);
+                // fsn 1: commit-bearing delta frame with deltaStart 10, deltaCount 0 --
+                // introduces NOTHING, but its baseline (10) sits ABOVE every id any
+                // surviving frame actually introduces (max 4). Models a commit /
+                // symbol-reusing frame emitted after ids 5..9 were registered by
+                // predecessor frames that have since been acked and trimmed away.
+                appendDeltaFrame(engine, false, 10, 0);
+            }
+            try (CursorSendEngine engine = newEngine()) {
+                assertTrue(engine.wasRecoveredFromDisk());
+                assertEquals("both frames are commit-bearing", 1L, engine.recoveredCommitBoundaryFsn());
+                // The zero-count frame drives recoveredMaxSymbolId to 9 (its baseline
+                // 10, minus 1), NOT to 4 (the highest id any surviving frame actually
+                // introduces) and NOT to 0 (which skipping it would yield). This
+                // inflation is exactly what makes seedGlobalDictionaryFromPersisted
+                // over-reject a dictionary holding ids 0..4 (size 5).
+                assertEquals("a zero-count delta frame anchors recoveredMaxSymbolId at its baseline-1",
+                        9L, engine.recoveredMaxSymbolId());
+            }
+        });
+    }
+
+    @Test
+    public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                appendDeltaSymbolFrame(engine, 0, 'a');
+                appendDeltaSymbolFrame(engine, 1, 'b');
+                appendDeltaSymbolFrame(engine, 2, 'c');
+            }
+
+            try (CursorSendEngine engine = newEngine()) {
+                assertEquals("one recovery visit per frame", 3L, engine.recoveryFramesVisited());
+
+                ObjList first = new ObjList<>();
+                ObjList second = new ObjList<>();
+                assertEquals(3L, engine.collectReplaySymbolsAbove(0, first));
+                assertEquals(3L, engine.collectReplaySymbolsAbove(0, second));
+                assertEquals("a", first.getQuick(0));
+                assertEquals("c", second.getQuick(2));
+                assertEquals("producer seed reads must reuse the recovery result",
+                        3L, engine.recoveryFramesVisited());
+
+                CursorWebSocketSendLoop loop = newLoop(engine, new ArrayList<>());
+                try {
+                    assertEquals("send-loop construction must copy the cached native suffix",
+                            3L, engine.recoveryFramesVisited());
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testRecoverySkipsEntriesAlreadyCoveredByPersistedPrefix() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                assertNotNull(pd);
+                pd.appendSymbol("a");
+                pd.appendSymbol("b");
+                pd.appendSymbol("c");
+                appendDeltaSymbolFrame(engine, 0, 'a');
+                appendDeltaSymbolFrame(engine, 1, 'b');
+                appendDeltaSymbolFrame(engine, 2, 'c');
+            }
+
+            try (CursorSendEngine engine = newEngine()) {
+                assertEquals("covered delta payloads must not be parsed entry-by-entry",
+                        0L, engine.recoverySymbolEntriesVisited());
+                GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+                PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                assertNotNull(pd);
+                pd.addLoadedSymbolsTo(recovered);
+                assertEquals(3L, engine.addRecoveredSymbolsTo(recovered.size(), recovered));
+                assertEquals("direct decode must not duplicate the covered frame entries",
+                        3, recovered.size());
+                assertEquals("c", recovered.getSymbol(2));
+            }
+        });
+    }
+
+    @Test
+    public void testSelfSufficientFrameRepairsAckedRecoveryGap() throws Exception {
+        // fsn 0 models the tail of an old delta epoch whose registering frames
+        // have already been trimmed: with an empty persisted dictionary its
+        // deltaStart=1 is a gap. It is durably ACKed, so it will not replay.
+        // fsn 1 starts a new, self-sufficient epoch from id 0 and is the first
+        // frame that WILL replay. Recovery must use that full frame as the new
+        // source of truth instead of permanently latching the earlier gap.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                appendDeltaFrame(engine, false, 1, 0);
+                appendDeltaSymbolFrame(engine, 0, 'a');
+            }
+            try (AckWatermark watermark = AckWatermark.open(tmpDir)) {
+                assertNotNull(watermark);
+                watermark.write(0L);
+            }
+
+            try (CursorSendEngine engine = newEngine()) {
+                assertEquals(0L, engine.ackedFsn());
+                ObjList recovered = new ObjList<>();
+                assertEquals("the full frame must re-anchor replay coverage",
+                        1L, engine.collectReplaySymbolsAbove(0, recovered));
+                assertEquals(1, recovered.size());
+                assertEquals("a", recovered.getQuick(0));
+            }
+        });
+    }
+
+    @Test
+    public void testSelfSufficientFrameCannotHideUnackedRecoveryGap() throws Exception {
+        // Safety twin: when the gapped frame itself is unacked, it reaches the
+        // wire before the later full frame. Recovery must keep the gap latched
+        // and quarantine rather than pretending the later reset repairs the
+        // invalid replay order.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                appendDeltaFrame(engine, false, 1, 0);
+                appendDeltaSymbolFrame(engine, 0, 'a');
+            }
+            try (CursorSendEngine engine = newEngine()) {
+                ObjList recovered = new ObjList<>();
+                assertEquals("an unacked gap remains unreplayable",
+                        -1L, engine.collectReplaySymbolsAbove(0, recovered));
+                assertEquals(0, recovered.size());
+            }
+        });
+    }
+
     // ---------------------------------------------------------------------
     // harness
     // ---------------------------------------------------------------------
@@ -291,6 +549,46 @@ private static void appendFrame(CursorSendEngine engine, boolean defer) {
         }
     }
 
+    // Appends a QWP frame carrying a symbol-dict delta section ([deltaStart varint]
+    // [deltaCount varint]) so the recovery walk's maxSymbolDeltaEnd counts it.
+    // deltaStart/deltaCount stay < 128 so each encodes in a single LEB128 byte.
+    private static void appendDeltaFrame(CursorSendEngine engine, boolean defer, int deltaStart, int deltaCount) {
+        int size = HEADER_SIZE + 2;
+        long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT);
+        try {
+            for (int i = 0; i < size; i++) {
+                Unsafe.getUnsafe().putByte(buf + i, (byte) 0);
+            }
+            Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE);
+            Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS,
+                    (byte) (FLAG_DELTA_SYMBOL_DICT | (defer ? FLAG_DEFER_COMMIT : 0)));
+            Unsafe.getUnsafe().putByte(buf + HEADER_SIZE, (byte) deltaStart);
+            Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 1, (byte) deltaCount);
+            engine.appendBlocking(buf, size);
+        } finally {
+            Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
+    private static void appendDeltaSymbolFrame(CursorSendEngine engine, int deltaStart, char symbol) {
+        int size = HEADER_SIZE + 4; // start, count=1, symbolLen=1, symbol byte
+        long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT);
+        try {
+            for (int i = 0; i < size; i++) {
+                Unsafe.getUnsafe().putByte(buf + i, (byte) 0);
+            }
+            Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE);
+            Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS, (byte) FLAG_DELTA_SYMBOL_DICT);
+            Unsafe.getUnsafe().putByte(buf + HEADER_SIZE, (byte) deltaStart);
+            Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 1, (byte) 1);
+            Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 2, (byte) 1);
+            Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 3, (byte) symbol);
+            engine.appendBlocking(buf, size);
+        } finally {
+            Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
     private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException {
         long deadline = System.nanoTime() + 10_000_000_000L;
         while (engine.ackedFsn() < target) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java
index 34ae5518..d63887db 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java
@@ -244,6 +244,77 @@ public void testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine() throws Exception
         });
     }
 
+    @Test
+    public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception {
+        // C2 regression: the dictionary catch-up advances nextWireSeq WITHOUT
+        // sending a data frame. A non-orderly close in that window -- a flapping
+        // LB/middlebox that completes the upgrade, accepts the catch-up, then drops
+        // before the first replay frame -- must be strike-EXEMPT. Keying the
+        // poison-strike gate off nextWireSeq > 0 (rather than
+        // dataFrameSentThisConnection) charges a strike on a frame that was never
+        // sent; MAX_REJECTIONS such closes then escalate a TRANSIENT outage to a
+        // PROTOCOL_VIOLATION terminal, hard-failing the producer and quarantining an
+        // orphan drainer -- exactly what store-and-forward's retry-forever contract
+        // forbids. Mirror image of testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine,
+        // which sends a real data frame (setSentCount) and DOES escalate.
+        TestUtils.assertMemoryLeak(() -> {
+            List clients = new ArrayList<>();
+            try (CursorSendEngine engine = newEngine()) {
+                appendFrames(engine, 2);
+                CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+                for (int i = 0; i < MAX_REJECTIONS + 2; i++) {
+                    // Model the catch-up: nextWireSeq advanced, but NO data frame
+                    // sent. swapClient resets both on every recycle, so re-apply
+                    // before each close. Pre-fix, each of these lands a strike on the
+                    // never-sent head frame and the loop terminals by now.
+                    setCatchUpWireSeqOnly(loop, 2);
+                    deliverNonOrderlyClose(loop);
+                }
+                // No strike was ever charged, so nothing escalated: the loop stays
+                // retriable and the producer-facing error latch is clear.
+                loop.checkError();
+            } finally {
+                closeAll(clients);
+            }
+        });
+    }
+
+    @Test
+    public void testNonOrderlyRejectionAfterOnlyCatchUpDoesNotStrike() throws Exception {
+        // C2 regression: the server-NACK twin of
+        // testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike. handleServerRejection's
+        // pre-send gate keys off dataFrameSentThisConnection, NOT nextWireSeq -- the
+        // dictionary catch-up advances nextWireSeq WITHOUT sending a data frame. A
+        // server NACK of a catch-up frame (nextWireSeq>0, no data frame sent) must
+        // take the pre-send path (surface + recycle, no strike), not the post-send
+        // poison-strike path. Pre-fix (gate on highestSent >= 0, i.e. nextWireSeq>0)
+        // each catch-up NACK strikes the never-sent head frame; MAX_REJECTIONS such
+        // strikes escalate a TRANSIENT outage to a producer-fatal PROTOCOL_VIOLATION
+        // terminal -- exactly what store-and-forward's retry-forever contract forbids.
+        // WRITE_ERROR (RETRIABLE) is the discriminating category: it DOES accrue
+        // strikes on the post-send path (see testNackRecycleIsPacedAgainstHealthyServer),
+        // so a regressed gate escalates here and checkError() throws.
+        TestUtils.assertMemoryLeak(() -> {
+            List clients = new ArrayList<>();
+            try (CursorSendEngine engine = newEngine()) {
+                appendFrames(engine, 2);
+                CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+                for (int i = 0; i < MAX_REJECTIONS + 2; i++) {
+                    // Model the catch-up: nextWireSeq advanced, but NO data frame
+                    // sent. The pre-send recycle (swapClient) resets nextWireSeq, so
+                    // re-apply before each NACK (mirrors the onClose twin).
+                    setCatchUpWireSeqOnly(loop, 2);
+                    deliverRetriableNack(loop, 1, "disk full");
+                }
+                // No strike was ever charged, so nothing escalated: the loop stays
+                // retriable and the producer-facing error latch is clear.
+                loop.checkError();
+            } finally {
+                closeAll(clients);
+            }
+        });
+    }
+
     @Test
     public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception {
         // A reachable, healthy server that NACKs the head frame (RETRIABLE)
@@ -585,6 +656,88 @@ public void testPoisonDwellHoldsEscalationUntilWallClockWindowElapses() throws E
         });
     }
 
+    @Test
+    public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Exception {
+        // Regression: after a reconnect the loop ships the head DATA frame
+        // (dataFrameSentThisConnection=true) BEFORE tryReceiveAcks reads the server's
+        // NACK of a dictionary CATCH-UP frame in the same loop iteration. That NACK
+        // names a wire seq below the replay head, so its fsn = fsnAtZero+cappedSeq is
+        // at or below ackedFsn -- negative when replayStart < catchUpFrames. It must
+        // NOT charge a poison strike: recordHeadRejectionStrike(fsn) would set
+        // poisonFsn to that value (the common shape yields -1, the "no suspect"
+        // sentinel), laundering any genuine in-progress poison run and reporting a
+        // bogus fsn. The fix routes an fsn <= ackedFsn rejection to the pre-send path
+        // (surface + recycle, no strike), symmetric with the success path's
+        // engine.acknowledge() no-op below ackedFsn.
+        TestUtils.assertMemoryLeak(() -> {
+            List clients = new ArrayList<>();
+            try (CursorSendEngine engine = newEngine()) {
+                appendFrames(engine, 2);
+                CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+                // Model: 2 catch-up frames (wire seq 0,1) + 1 data frame (wire seq 2)
+                // sent, nothing acked (ackedFsn = -1), so fsnAtZero = replayStart(0) -
+                // catchUpFrames(2) = -2. A NACK of catch-up wire seq 0 maps to fsn -2,
+                // strictly below the -1 sentinel so the assertion discriminates.
+                setPostCatchUpDataFrameBaseline(loop, -2L, 3L);
+                assertEquals("precondition: no poison suspect yet",
+                        -1L, getLongField(loop, "poisonFsn"));
+
+                deliverRetriableNack(loop, 0, "disk full");
+
+                // The catch-up NACK was surfaced + recycled but charged NO strike, so
+                // the sentinel is intact. Pre-fix it was set to -2, laundering the
+                // poison detector's state.
+                assertEquals("catch-up NACK must not set/launder the poison sentinel",
+                        -1L, getLongField(loop, "poisonFsn"));
+                loop.checkError();
+            } finally {
+                closeAll(clients);
+            }
+        });
+    }
+
+    @Test
+    public void testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike() throws Exception {
+        // Boundary companion to testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState:
+        // the COMMON single-frame catch-up maps its only catch-up frame to
+        // fsn == ackedFsn EXACTLY (the last catch-up frame lands on replayStart-1 ==
+        // ackedFsn). The pre-send routing guard is "fsn <= ackedFsn", so this equality
+        // case must also skip the poison strike. Without pinning it a "<=" -> "<"
+        // mutation would slip the single/last catch-up frame's NACK onto the post-send
+        // poison-strike path and launder the detector -- exactly the hazard the guard
+        // closes -- yet the strictly-below test above would still pass.
+        TestUtils.assertMemoryLeak(() -> {
+            List clients = new ArrayList<>();
+            try (CursorSendEngine engine = newEngine()) {
+                appendFrames(engine, 2);
+                CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+                // 1 catch-up frame (wire seq 0) + 1 data frame (wire seq 1), nothing
+                // acked (ackedFsn = -1), so fsnAtZero = replayStart(0) - catchUpFrames(1)
+                // = -1. A NACK of catch-up wire seq 0 maps to fsn -1 == ackedFsn exactly.
+                setPostCatchUpDataFrameBaseline(loop, -1L, 2L);
+                assertEquals("precondition: no strikes yet",
+                        0L, getLongField(loop, "poisonStrikes"));
+
+                deliverRetriableNack(loop, 0, "disk full");
+
+                // poisonStrikes -- NOT poisonFsn -- is the discriminator at this exact
+                // boundary: a wrongly-charged strike calls recordHeadRejectionStrike(-1),
+                // which sets poisonFsn to the rejected fsn -1 -- the "no suspect"
+                // sentinel -- so a poisonFsn check cannot see the laundering here. It
+                // still bumps poisonStrikes 0 -> 1, so the correct pre-send routing (no
+                // strike) shows as poisonStrikes staying 0. A "<=" -> "<" mutation of the
+                // fsn-vs-ackedFsn guard charges the strike and fails this.
+                assertEquals("catch-up NACK at fsn == ackedFsn must not charge a poison strike",
+                        0L, getLongField(loop, "poisonStrikes"));
+                assertEquals("... and must leave the poison sentinel intact",
+                        -1L, getLongField(loop, "poisonFsn"));
+                loop.checkError();
+            } finally {
+                closeAll(clients);
+            }
+        });
+    }
+
     @Test
     public void testPostSendNotWritableNackNeverEscalatesToPoisonTerminal() throws Exception {
         // RETRIABLE_OTHER (NOT_WRITABLE) is a node-state verdict, not a frame
@@ -969,6 +1122,44 @@ private static void setSentCount(CursorWebSocketSendLoop loop, long count) throw
         Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq");
         f.setAccessible(true);
         f.setLong(loop, count);
+        // A non-zero sent count models DATA frames sent on this connection. The
+        // poison-strike / pre-send gates key off dataFrameSentThisConnection, not
+        // nextWireSeq (the dictionary catch-up advances nextWireSeq without sending
+        // a data frame), so keep the two in sync for these white-box scenarios.
+        Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection");
+        d.setAccessible(true);
+        d.setBoolean(loop, count > 0);
+    }
+
+    // Sets ONLY nextWireSeq -- deliberately NOT dataFrameSentThisConnection -- to
+    // model the dictionary catch-up having advanced the wire sequence with no data
+    // frame sent yet. Contrast setSentCount, which sets both.
+    private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long count) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq");
+        f.setAccessible(true);
+        f.setLong(loop, count);
+    }
+
+    // Models a fresh connection that has shipped the dictionary catch-up (advancing
+    // fsnAtZero below the replay head) AND the head data frame: sets fsnAtZero,
+    // nextWireSeq, and dataFrameSentThisConnection=true together.
+    private static void setPostCatchUpDataFrameBaseline(CursorWebSocketSendLoop loop,
+                                                        long fsnAtZero, long nextWireSeq) throws Exception {
+        Field z = CursorWebSocketSendLoop.class.getDeclaredField("fsnAtZero");
+        z.setAccessible(true);
+        z.setLong(loop, fsnAtZero);
+        Field w = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq");
+        w.setAccessible(true);
+        w.setLong(loop, nextWireSeq);
+        Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection");
+        d.setAccessible(true);
+        d.setBoolean(loop, true);
+    }
+
+    private static long getLongField(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getLong(loop);
     }
 
     private static long[] txns(long... v) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
new file mode 100644
index 00000000..5a666dc7
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
@@ -0,0 +1,221 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.DefaultHttpClientConfiguration;
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.network.PlainSocketFactory;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * White-box coverage of the send loop's torn-dictionary guard
+ * ({@code trySendOne}: {@code deltaStart > sentDictCount}) firing on a genuine gap.
+ * 

+ * That guard is the drainer path's SOLE defense: an orphan drainer adopts a slot
+ * without running {@code Sender.build()}'s seed-time guard, so on a slot whose
+ * registering frames were trimmed away and whose {@code .symbol-dict} was torn, the
+ * send loop must detect the gap itself, ship ZERO frames, and latch a terminal --
+ * never null-pad the hole on the server. The foreground sender is always quarantined
+ * earlier (at build time), so this fire direction runs only here. Driven at the send
+ * loop level against a frame-counting stub client so it exercises the guard
+ * deterministically, without a real network connection.
+ */
+public class CursorWebSocketSendLoopTornDictGuardTest {
+
+    private String sfDir;
+
+    @Before
+    public void setUp() {
+        sfDir = TestUtils.createTmpDir("qdb-torn-dict-guard-");
+    }
+
+    @After
+    public void tearDown() {
+        TestUtils.removeTmpDir(sfDir);
+    }
+
+    @Test
+    public void testGuardFiresOnGenuineGapAndShipsNoFrame() throws Exception {
+        assertMemoryLeak(() -> {
+            writeAndTearGappedSlot();
+
+            CountingClient client = new CountingClient();
+            try (CursorSendEngine engine = new CursorSendEngine(sfDir + "/default", 16_384)) {
+                // The persisted dict was torn away and the registering frames trimmed, so
+                // the mirror cannot be seeded from either source: sentDictCount stays 0
+                // while the first surviving frame's delta starts above it.
+                CursorWebSocketSendLoop loop = newLoop(engine, client);
+                try {
+                    setRunning(loop);
+                    // Position at the first unsent frame exactly as the I/O loop does
+                    // before its first send (no catch-up: the mirror is empty).
+                    invoke(loop, "positionCursorForStart");
+
+                    boolean sent = (Boolean) invoke(loop, "trySendOne");
+
+                    Assert.assertFalse("the torn-dict guard must refuse to send the gapped frame", sent);
+                    Assert.assertEquals("no frame may reach the server through a gap",
+                            0, client.framesSent);
+                    try {
+                        loop.checkError();
+                        Assert.fail("the guard must latch a terminal error");
+                    } catch (LineSenderException e) {
+                        Assert.assertTrue("unexpected terminal: " + e.getMessage(),
+                                e.getMessage().contains("incomplete")
+                                        && e.getMessage().contains("resend required"));
+                    }
+                } finally {
+                    loop.close();
+                }
+            }
+        });
+    }
+
+    private static int countSegmentFiles(Path dir) {
+        File[] files = dir.toFile().listFiles();
+        int n = 0;
+        if (files != null) {
+            for (File f : files) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")) {
+                    n++;
+                }
+            }
+        }
+        return n;
+    }
+
+    private static Object invoke(CursorWebSocketSendLoop loop, String method) throws Exception {
+        Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(method);
+        m.setAccessible(true);
+        return m.invoke(loop);
+    }
+
+    private static void setRunning(CursorWebSocketSendLoop loop) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField("running");
+        f.setAccessible(true);
+        f.setBoolean(loop, true);
+    }
+
+    // Constructs a recovery send loop that is never started -- the test drives
+    // positionCursorForStart + trySendOne directly. The reconnect factory throws
+    // because no reconnect is expected before the guard latches its terminal.
+    private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) {
+        return new CursorWebSocketSendLoop(
+                client, engine, 0, 1_000_000L,
+                () -> {
+                    throw new IOException("no reconnect in this test");
+                },
+                0, 0, 1);
+    }
+
+    // Writes 12 delta frames (each a new symbol) into the default slot across several
+    // small segments, then makes the slot carry a GENUINE gap: trims the segment
+    // holding the earliest ids (munmap + unlink, as SegmentManager does once they are
+    // acked) and tears the .symbol-dict down to its header. The surviving frames' deltas
+    // then start above ids nothing on disk still holds, so the mirror cannot be rebuilt.
+    private void writeAndTearGappedSlot() throws Exception {
+        try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+            int port = silent.getPort();
+            silent.start();
+            Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+            String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+                    + ";sf_max_bytes=256;close_flush_timeout_millis=0;";
+            try (Sender s = Sender.fromConfig(cfg)) {
+                for (int i = 0; i < 12; i++) {
+                    s.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+                    s.flush();
+                }
+            }
+        }
+        Path slot = Paths.get(sfDir, "default");
+        Assert.assertTrue("the frames must have rolled into more than one segment",
+                countSegmentFiles(slot) > 1);
+        Files.delete(slot.resolve("sf-initial.sfa"));
+        Path dict = slot.resolve(".symbol-dict");
+        Files.write(dict, Arrays.copyOf(Files.readAllBytes(dict), 8));
+    }
+
+    // Frame-counting stub transport: completes no real I/O. If the guard ever lets a
+    // gapped frame through, sendBinary bumps framesSent and the test fails.
+    private static final class CountingClient extends WebSocketClient {
+        private int framesSent;
+
+        CountingClient() {
+            super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+        }
+
+        @Override
+        public int getServerMaxBatchSize() {
+            return 16_384;
+        }
+
+        @Override
+        public int getServerQwpVersion() {
+            return 1;
+        }
+
+        @Override
+        public void sendBinary(long dataPtr, int length) {
+            framesSent++;
+        }
+
+        @Override
+        protected void ioWait(int timeout, int op) {
+        }
+
+        @Override
+        protected void setupIoWait() {
+        }
+    }
+
+    private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler {
+        @Override
+        public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+            // never acks -- the sender leaves everything unacked in the slot
+        }
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java
index 5c7607a6..2efee2e4 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java
@@ -25,6 +25,7 @@
 package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
 
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
 import io.questdb.client.std.Files;
 import io.questdb.client.test.tools.TestUtils;
 import org.junit.After;
@@ -35,6 +36,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 
 /**
  * Regression test for M6 — drainer adopting an empty orphan slot would
@@ -87,6 +89,48 @@ public void tearDown() {
         Files.remove(sfDir);
     }
 
+    @Test
+    public void testFreshStartDiscardsSurvivingStaleDictionary() throws Exception {
+        // Regression: a prior fully-drained lifecycle can leave a stale
+        // .symbol-dict behind (a best-effort delete that failed, or a crash in the
+        // close window) with NO segments. A fresh start must DISCARD it -- the
+        // dictionary is load-bearing and the fresh-start producer is not seeded
+        // from it, so trusting a survivor would diverge the producer ids from the
+        // dictionary the send loop replays and misattribute symbols on reconnect.
+        TestUtils.assertMemoryLeak(() -> {
+            // Pre-seed a stale dictionary in the slot, with no segments behind it.
+            PersistedSymbolDict stale = PersistedSymbolDict.open(sfDir);
+            assertNotNull(stale);
+            try {
+                stale.appendSymbol("staleX");
+                stale.appendSymbol("staleY");
+                assertEquals(2, stale.size());
+            } finally {
+                stale.close();
+            }
+
+            // A fresh start (no recovered segments) must open a CLEAN, empty
+            // dictionary -- not inherit the survivor.
+            try (CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024)) {
+                assertFalse("fresh start must not report a disk recovery",
+                        engine.wasRecoveredFromDisk());
+                PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                assertNotNull(pd);
+                assertEquals("fresh start must discard the surviving stale dictionary",
+                        0, pd.size());
+            }
+
+            // The survivor's bytes are physically gone, not just hidden.
+            PersistedSymbolDict reopened = PersistedSymbolDict.open(sfDir);
+            assertNotNull(reopened);
+            try {
+                assertEquals(0, reopened.size());
+            } finally {
+                reopened.close();
+            }
+        });
+    }
+
     @Test
     public void testNeverPublishedCloseLeavesNoSfaFiles() throws Exception {
         TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
index 177ee5f6..da9c4494 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
@@ -35,6 +35,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.RandomAccessFile;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -98,6 +100,8 @@ public void testCreateAppendCloseReopenScansAllFrames() throws Exception {
                     long expectedEnd = MmapSegment.HEADER_SIZE
                             + 100L * (MmapSegment.FRAME_HEADER_SIZE + 32);
                     assertEquals(expectedEnd, seg.publishedOffset());
+                    assertEquals("validation scan must return the frame count",
+                            100L, seg.frameCount());
                 }
 
                 // Re-open: scan must land at exactly the same offset.
@@ -113,6 +117,31 @@ public void testCreateAppendCloseReopenScansAllFrames() throws Exception {
         });
     }
 
+    @Test
+    public void testCurrentReaderStillOpensLegacyV1Segment() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            String path = tmpDir + "/seg-v1.sfa";
+            long buf = Unsafe.malloc(8, MemoryTag.NATIVE_DEFAULT);
+            try {
+                try (MmapSegment seg = MmapSegment.create(path, 7L, 4096)) {
+                    assertEquals(MmapSegment.VERSION, seg.version());
+                    assertTrue(seg.tryAppend(buf, 8) >= 0L);
+                }
+                try (RandomAccessFile file = new RandomAccessFile(path, "rw")) {
+                    file.seek(4L);
+                    file.writeByte(MmapSegment.LEGACY_VERSION);
+                }
+                try (MmapSegment legacy = MmapSegment.openExisting(path)) {
+                    assertEquals(MmapSegment.LEGACY_VERSION, legacy.version());
+                    assertEquals(7L, legacy.baseSeq());
+                    assertEquals(1L, legacy.frameCount());
+                }
+            } finally {
+                Unsafe.free(buf, 8, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception {
         TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java
index 162474fe..a39bef67 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
 
+import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
 import io.questdb.client.std.Files;
 import io.questdb.client.std.IntList;
@@ -321,6 +322,30 @@ public void testIsCandidateOrphanDirect() throws Exception {
         });
     }
 
+    @Test
+    public void testDrainerRevalidatesStaleScannerSnapshotBeforeCreatingEngine() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            String slot = sfDir + "/stale";
+            assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT));
+            String segment = slot + "/sf-0001.sfa";
+            touchFile(segment);
+            assertTrue(OrphanScanner.isCandidateOrphan(slot));
+
+            // The scanner has already queued this pathname, but another owner
+            // drains/removes its last segment before the worker gets to run.
+            assertTrue(Files.remove(segment));
+            BackgroundDrainer drainer = new BackgroundDrainer(
+                    slot, 1024, 8192, () -> {
+                throw new AssertionError("a stale candidate must not connect");
+            }, 1000, 1, 10, true, 0);
+            drainer.run();
+
+            assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome());
+            assertFalse("stale adoption must not create the directory-local lock",
+                    Files.exists(slot + "/.lock"));
+        });
+    }
+
     private static void touchFile(String path) {
         int fd = Files.openRW(path);
         if (fd >= 0) Files.close(fd);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
new file mode 100644
index 00000000..ee046d33
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
@@ -0,0 +1,880 @@
+/*+*****************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.test.tools.DelegatingFilesFacade;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+public class PersistedSymbolDictTest {
+
+    // On-disk geometry, mirrored from PersistedSymbolDict so the layout-derived
+    // corruption tests below read as arithmetic rather than magic numbers.
+    private static final int CRC_SIZE = 4;
+    private static final int HEADER_SIZE = 8;
+
+    @Rule
+    public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+    @Test
+    public void testAppendPersistsAcrossReopen() throws Exception {
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(d);
+                Assert.assertEquals(0, d.size());
+                d.appendSymbol("AAPL");
+                d.appendSymbol("GOOG");
+                d.appendSymbol("MSFT");
+                Assert.assertEquals(3, d.size());
+            }
+
+            // Reopen: entries recovered in id order.
+            try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(reopened);
+                Assert.assertEquals(3, reopened.size());
+                ObjList symbols = reopened.readLoadedSymbols();
+                Assert.assertEquals(3, symbols.size());
+                Assert.assertEquals("AAPL", symbols.getQuick(0));
+                Assert.assertEquals("GOOG", symbols.getQuick(1));
+                Assert.assertEquals("MSFT", symbols.getQuick(2));
+                Assert.assertTrue(reopened.loadedEntriesLen() > 0);
+                Assert.assertTrue("production recovery must parse through mmap instead of a heap file copy",
+                        reopened.usedMappedRecoveryInput());
+
+                GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+                reopened.addLoadedSymbolsTo(recovered);
+                Assert.assertEquals("direct recovery decode must preserve dense ids", 3, recovered.size());
+                Assert.assertEquals("GOOG", recovered.getSymbol(1));
+
+                // Appending after recovery continues from the recovered tip.
+                reopened.appendSymbol("TSLA");
+                Assert.assertEquals(4, reopened.size());
+            }
+
+            try (PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertEquals(4, third.size());
+                Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3));
+            }
+        });
+    }
+
+    @Test
+    public void testAppendRawEntriesMatchesAppendSymbols() throws Exception {
+        // M1: the producer persists the frame's already-encoded delta bytes via
+        // appendRawEntries instead of re-encoding the symbols. Those bytes are the
+        // same [len][utf8]... layout appendSymbols writes, so both must produce an
+        // identical, recoverable dictionary. Encode a range with appendSymbols,
+        // reopen to grab its on-disk entry bytes, replay them through
+        // appendRawEntries into a fresh dict, and assert the recovered symbols
+        // match -- including an empty entry mid-range.
+        assertMemoryLeak(() -> {
+            Path src = newFolder("qwp-symdict-src");
+            Path dst = newFolder("qwp-symdict-dst");
+            GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+            dict.getOrAddSymbol("AAPL"); // id 0
+            dict.getOrAddSymbol("");     // id 1 -- empty entry mid-range
+            dict.getOrAddSymbol("MSFT"); // id 2
+
+            try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) {
+                encoded.appendSymbols(dict, 0, 2);
+            }
+
+            // Reopen to obtain the on-disk entry region [len][utf8]... verbatim,
+            // then replay it byte-for-byte into a fresh dict via appendRawEntries.
+            try (PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString());
+                 PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString())) {
+                raw.appendRawEntries(reopened.loadedEntriesAddr(),
+                        reopened.loadedEntriesLen(), reopened.size());
+                Assert.assertEquals(3, raw.size());
+            }
+
+            // The raw-appended dict must recover the same dense symbols.
+            try (PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString())) {
+                Assert.assertEquals(3, recovered.size());
+                ObjList symbols = recovered.readLoadedSymbols();
+                Assert.assertEquals("AAPL", symbols.getQuick(0));
+                Assert.assertEquals("", symbols.getQuick(1));
+                Assert.assertEquals("MSFT", symbols.getQuick(2));
+            }
+        });
+    }
+
+    @Test
+    public void testAppendRawEntriesRejectsInconsistentTripleUnderAssertions() throws Exception {
+        // appendRawEntries validates the (addr,len,count) triple before writing: an
+        // inconsistent triple would record a chunk whose stored entryCount disagreed
+        // with its entries and shift the dense id->symbol map on recovery, so it must
+        // fail loudly rather than corrupt the file. That check is an assert -- gated on
+        // -ea to keep the per-entry walk off the per-flush production path (client apps
+        // run without -ea) -- so observe it under the test suite's -ea, and skip if
+        // assertions happen to be disabled.
+        boolean assertionsEnabled = false;
+        assert assertionsEnabled = true;
+        Assume.assumeTrue("the guard is assert-gated; only observable under -ea", assertionsEnabled);
+        assertMemoryLeak(() -> {
+            Path src = newFolder("qwp-symdict-src");
+            Path dst = newFolder("qwp-symdict-dst");
+            GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+            dict.getOrAddSymbol("AAPL"); // id 0
+            dict.getOrAddSymbol("MSFT"); // id 1
+            dict.getOrAddSymbol("GOOG"); // id 2
+            try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) {
+                encoded.appendSymbols(dict, 0, 2);
+            }
+
+            // Grab the on-disk entry region (3 entries) verbatim, then feed it back
+            // with a count of 1: validateRawEntries stops one entry in with
+            // src < srcLimit and throws "under-filled", before any bytes are written.
+            try (PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString());
+                 PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString())) {
+                try {
+                    raw.appendRawEntries(reopened.loadedEntriesAddr(), reopened.loadedEntriesLen(), 1);
+                    Assert.fail("an inconsistent (len,count) triple must be rejected");
+                } catch (IllegalStateException expected) {
+                    Assert.assertTrue("message names the under-filled buffer: " + expected.getMessage(),
+                            expected.getMessage().contains("under-filled"));
+                    Assert.assertEquals("a rejected triple must write nothing", 0, raw.size());
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testAppendRawEntriesShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception {
+        // The producer's FAST path persists a frame's already-encoded delta bytes
+        // via appendRawEntries. A short write (disk full / quota) mid-persist must
+        // throw WITHOUT advancing size()/appendOffset, so a retry keyed off size()
+        // re-writes the identical bytes at the same offset and recovers a gap-free,
+        // duplicate-free dictionary. The failed-PUBLISH regressions
+        // (DeltaDictRecoveryTest) exercise a persist that SUCCEEDED; only this pins
+        // the persist-FAILURE trigger, whose idempotency the write-ahead
+        // (persistNewSymbolsBeforePublish, resuming from pd.size()) relies on.
+        assertMemoryLeak(() -> {
+            Path src = newFolder("qwp-symdict-src");
+            Path dst = newFolder("qwp-symdict-dst");
+            GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+            dict.getOrAddSymbol("AAPL"); // id 0
+            dict.getOrAddSymbol("MSFT"); // id 1
+
+            // Encode the range once to obtain its on-disk [len][utf8]... bytes.
+            try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) {
+                encoded.appendSymbols(dict, 0, 1);
+            }
+
+            try (PersistedSymbolDict source = PersistedSymbolDict.open(src.toString())) {
+                long addr = source.loadedEntriesAddr();
+                int rawLen = source.loadedEntriesLen();
+                int count = source.size();
+
+                ShortWriteOnceFacade ff = new ShortWriteOnceFacade();
+                try (PersistedSymbolDict d = PersistedSymbolDict.open(ff, dst.toString())) {
+                    Assert.assertNotNull(d);
+                    ff.armed = true; // the next entry append lands short
+                    try {
+                        d.appendRawEntries(addr, rawLen, count);
+                        Assert.fail("a short write must throw");
+                    } catch (IllegalStateException expected) {
+                        Assert.assertTrue("short-write error: " + expected.getMessage(),
+                                expected.getMessage().contains("short write"));
+                    }
+                    // The throw preceded the size/offset advance: nothing persisted.
+                    Assert.assertEquals("short write must NOT advance size", 0, d.size());
+
+                    // Retry the SAME bytes (the facade auto-disarmed): the write
+                    // lands at the unchanged offset, overwriting the torn prefix.
+                    d.appendRawEntries(addr, rawLen, count);
+                    Assert.assertEquals(2, d.size());
+                }
+            }
+
+            // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3).
+            try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dst.toString())) {
+                Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size());
+                ObjList symbols = reopened.readLoadedSymbols();
+                Assert.assertEquals("AAPL", symbols.getQuick(0));
+                Assert.assertEquals("MSFT", symbols.getQuick(1));
+            }
+        });
+    }
+
+    @Test
+    public void testAppendSymbolsBatchWritesDenseRange() throws Exception {
+        // appendSymbols persists a whole id range in one write (the hot-path
+        // syscall reduction). It must produce the same dense, id-ordered file a
+        // per-symbol loop would, including an empty symbol mid-range, and a second
+        // batched call keyed off size() must continue densely (the resume-from-
+        // durable-size contract the producer relies on).
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+            dict.getOrAddSymbol("AAPL"); // id 0
+            dict.getOrAddSymbol("");     // id 1 -- empty symbol mid-range
+            dict.getOrAddSymbol("MSFT"); // id 2
+            dict.getOrAddSymbol("TSLA"); // id 3
+
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbols(dict, 0, 3); // one write for all four ids
+                Assert.assertEquals(4, d.size());
+                d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op
+                Assert.assertEquals(4, d.size());
+            }
+
+            try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertEquals(4, reopened.size());
+                ObjList symbols = reopened.readLoadedSymbols();
+                Assert.assertEquals(4, symbols.size());
+                Assert.assertEquals("AAPL", symbols.getQuick(0));
+                Assert.assertEquals("", symbols.getQuick(1));
+                Assert.assertEquals("MSFT", symbols.getQuick(2));
+                Assert.assertEquals("TSLA", symbols.getQuick(3));
+
+                // A follow-on batch keyed off the recovered size continues
+                // the dense sequence without a gap or duplicate.
+                dict.getOrAddSymbol("NVDA"); // id 4
+                reopened.appendSymbols(dict, reopened.size(), 4);
+                Assert.assertEquals(5, reopened.size());
+            }
+
+            try (PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertEquals(5, third.size());
+                Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4));
+            }
+        });
+    }
+
+    @Test
+    public void testAppendSymbolsShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception {
+        // The producer's SLOW path (re-encode the [pd.size()..batchMax] suffix after
+        // a prior partial persist) writes via appendSymbols. A short write (disk full
+        // / quota) must throw WITHOUT advancing size()/appendOffset, so a retry keyed
+        // off size() re-persists the SAME range at the SAME offset and recovers a
+        // gap-free, duplicate-free dictionary. A regression that advanced size before
+        // the written==len check would strand a torn/duplicated dictionary here.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+            dict.getOrAddSymbol("AAPL"); // id 0
+            dict.getOrAddSymbol("GOOG"); // id 1
+
+            ShortWriteOnceFacade ff = new ShortWriteOnceFacade();
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(ff, dir.toString())) {
+                Assert.assertNotNull(d);
+                ff.armed = true; // the next entry append lands short
+                try {
+                    d.appendSymbols(dict, 0, 1);
+                    Assert.fail("a short write must throw");
+                } catch (IllegalStateException expected) {
+                    Assert.assertTrue("short-write error: " + expected.getMessage(),
+                            expected.getMessage().contains("short write"));
+                }
+                // The throw preceded the size/offset advance: nothing persisted.
+                Assert.assertEquals("short write must NOT advance size", 0, d.size());
+
+                // Retry the SAME range (the facade auto-disarmed): re-writes at
+                // the unchanged offset, overwriting the torn bytes.
+                d.appendSymbols(dict, 0, 1);
+                Assert.assertEquals(2, d.size());
+            }
+
+            // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3).
+            try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size());
+                ObjList symbols = reopened.readLoadedSymbols();
+                Assert.assertEquals("AAPL", symbols.getQuick(0));
+                Assert.assertEquals("GOOG", symbols.getQuick(1));
+            }
+        });
+    }
+
+    @Test
+    public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception {
+        // A batch append must write ONE chunk -- one [entryCount][entryBytes] header and
+        // ONE trailing CRC-32C for the whole range -- not one checksummed record per
+        // symbol.
+        //
+        // This is the load-bearing property of the v3 layout, and it is a producer-thread
+        // cost, not a cosmetic one: Crc32c.update is a NATIVE call, so a per-entry
+        // checksum put one JNI transition (plus one sub-cache-line copy and one redundant
+        // varint decode) on the flush path for every new symbol. On the high-cardinality
+        // batch this whole feature exists to serve -- one new symbol per row -- that is a
+        // thousand native calls per flush where the chunk needs one.
+        //
+        // Asserted through the file size, which is exact and cheap: N single-byte ASCII
+        // symbols cost N * ([len varint] + 1 utf8 byte) of entries, wrapped in one chunk.
+        // Per-entry CRCs would add 4 * N bytes instead of 4.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            final int n = 26; // 'a'..'z' -- distinct, and one UTF-8 byte each
+            GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+            for (int i = 0; i < n; i++) {
+                dict.getOrAddSymbol(String.valueOf((char) ('a' + i)));
+            }
+            Assert.assertEquals(n, dict.size());
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(d);
+                d.appendSymbols(dict, 0, n - 1);
+                Assert.assertEquals(n, d.size());
+            }
+
+            int entryBytes = n * 2;                       // [len=1 varint][1 utf8 byte]
+            int chunkHeader = 1 + varintSize(entryBytes); // entryCount=50 fits one byte
+            long expected = HEADER_SIZE + chunkHeader + entryBytes + CRC_SIZE;
+            Assert.assertEquals(
+                    "a batch append must cost ONE chunk checksum, not one per symbol",
+                    expected, Files.size(dir.resolve(".symbol-dict")));
+
+            // And it must still read back symbol-for-symbol.
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(re);
+                Assert.assertEquals(n, re.size());
+                ObjList got = re.readLoadedSymbols();
+                for (int i = 0; i < n; i++) {
+                    Assert.assertEquals(dict.getSymbol(i), got.getQuick(i));
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testMappedAppendAmortizesFlushesWithoutPositionedWrites() throws Exception {
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(d);
+                for (int i = 0; i < 10_000; i++) {
+                    d.appendSymbol("sym-" + i);
+                }
+                Assert.assertEquals(10_000, d.size());
+                Assert.assertEquals("production appends must write directly into the mmap",
+                        0L, d.appendWriteCount());
+                Assert.assertEquals("ten thousand flushes must require only three mapped windows",
+                        3, d.appendMapGrowthCount());
+            }
+
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(re);
+                Assert.assertEquals(10_000, re.size());
+                Assert.assertEquals("sym-9999", re.readLoadedSymbols().getQuick(9_999));
+            }
+        });
+    }
+
+    private static int varintSize(int v) {
+        int n = 1;
+        while ((v >>>= 7) != 0) {
+            n++;
+        }
+        return n;
+    }
+
+    @Test
+    public void testBadMagicDegradesWithoutDestroyingTheFile() throws Exception {
+        // An unparseable existing file must degrade to null -- the sender falls back
+        // to full self-sufficient frames -- and must NOT be recreated. open() used to
+        // fall through to openFresh(), which is O_TRUNC: a single unreadable byte
+        // destroyed the only copy of load-bearing state.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            Path f = dir.resolve(".symbol-dict");
+            byte[] original = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+            Files.write(f, original);
+            Assert.assertNull("an unparseable existing dict must degrade to null",
+                    PersistedSymbolDict.open(dir.toString()));
+            Assert.assertArrayEquals("open() must NOT destroy an existing dictionary",
+                    original, Files.readAllBytes(f));
+        });
+    }
+
+    @Test
+    public void testBadVersionDegradesWithoutDestroyingTheFile() throws Exception {
+        // A file with correct 'SYD1' magic and a VALID chunk but an unknown version
+        // byte belongs to a foreign/future format and must not be parsed as v3.
+        // Covers the version sub-condition specifically (testBadMagicDegrades... covers
+        // the magic one).
+        //
+        // The file must SURVIVE. This is the client-rollback trap: a newer client
+        // writes v4, ops roll back to this build, and if open() recreated the file the
+        // v4 dictionary would be gone for good -- rolling forward again could not
+        // recover it, and every surviving delta frame would be permanently
+        // unreplayable. Degrading to null instead leaves the bytes for the client that
+        // does understand them.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            Path f = dir.resolve(".symbol-dict");
+            try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(seed);
+                seed.appendSymbol("a");
+            }
+            // Corrupt ONLY the version byte (offset 4) to an unknown value.
+            byte[] bytes = Files.readAllBytes(f);
+            bytes[4] = (byte) 99;
+            Files.write(f, bytes);
+
+            Assert.assertNull("an unknown-version dict must degrade to null",
+                    PersistedSymbolDict.open(dir.toString()));
+            Assert.assertArrayEquals("open() must NOT destroy a future-version dictionary",
+                    bytes, Files.readAllBytes(f));
+        });
+    }
+
+    @Test
+    public void testCloseNullsLoadedEntries() throws Exception {
+        // close() must null loadedEntriesAddr/Len after freeing them (like
+        // scratchAddr), so an accidental post-close read of the getters cannot
+        // dereference freed native memory. Pre-fix the pointer survived close()
+        // non-zero.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("AAPL");
+            }
+
+            // Reopen so recovery loads the entries into native memory. This test
+            // closes explicitly because the post-close state is the assertion target.
+            PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString());
+            Assert.assertTrue("recovery must load entries into native memory",
+                    re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0);
+            re.close();
+            Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr());
+            Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen());
+        });
+    }
+
+    @Test
+    public void testEmptySymbolRoundTrips() throws Exception {
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("");
+                d.appendSymbol("nonempty");
+            }
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertEquals(2, re.size());
+                ObjList s = re.readLoadedSymbols();
+                Assert.assertEquals("", s.getQuick(0));
+                Assert.assertEquals("nonempty", s.getQuick(1));
+            }
+        });
+    }
+
+    @Test
+    public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exception {
+        // A host-crash interior tear (a lost page reading back as zeroes) or a
+        // stale chunk left past the end by a failed truncate can change the bytes
+        // of a NON-trailing chunk. Without the per-chunk CRC the parse would
+        // accept those bytes, shifting the dense id->symbol map and silently
+        // misattributing symbol-column values on replay. With the CRC the corrupt
+        // chunk fails verification and the parse stops there, so recovery trusts
+        // only the intact prefix (fail-clean: the send loop's torn-dict guard then
+        // forces a resend of the rest).
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("s0");
+                d.appendSymbol("s1");
+                d.appendSymbol("s2");
+                d.appendSymbol("s3");
+                d.appendSymbol("s4");
+                Assert.assertEquals(5, d.size());
+            }
+
+            // Corrupt one byte inside the 3rd chunk's entry region (id 2). Each
+            // appendSymbol writes one single-entry chunk
+            // ([entryCount][entryBytes][len][utf8][crc32c]), and all five symbols
+            // are the same width, so the five chunks are equal-sized -- derive the
+            // stride from the file rather than hard-coding it. The byte flipped is
+            // the last one before the chunk's trailing CRC, so that CRC goes stale.
+            Path f = dir.resolve(".symbol-dict");
+            byte[] bytes = Files.readAllBytes(f);
+            int chunkLen = (bytes.length - HEADER_SIZE) / 5;
+            int chunk2 = HEADER_SIZE + 2 * chunkLen;
+            bytes[chunk2 + chunkLen - CRC_SIZE - 1] ^= 0x7F;
+            Files.write(f, bytes);
+
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(re);
+                // Only the intact prefix [s0, s1] is trusted; the corrupt chunk
+                // and everything after it are dropped. No recovered symbol is
+                // the corrupted string -- the tear is DETECTED, never silently
+                // misattributed.
+                Assert.assertEquals("parse must stop at the corrupt interior chunk", 2, re.size());
+                ObjList s = re.readLoadedSymbols();
+                Assert.assertEquals("s0", s.getQuick(0));
+                Assert.assertEquals("s1", s.getQuick(1));
+            }
+        });
+    }
+
+    @Test
+    public void testLargeSymbolRoundTripsAcrossReopen() throws Exception {
+        // C1 regression: the write path caps nothing, so a symbol larger than the
+        // old fixed 1 MB read ceiling must still recover intact. Before the fix,
+        // appendSymbol wrote the oversized entry but openExisting rejected it as
+        // "oversized", truncated the dictionary at that id (dropping it and every
+        // higher id), and a normal process-crash recovery then hard-failed with a
+        // spurious "host crash / resend required" terminal -- defeating store-and-
+        // forward's process-crash durability for large symbols. The file length is
+        // now the only bound, so the write and read paths agree.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            // Just over the old 1 << 20 (1 MB) ceiling.
+            String big = TestUtils.repeat("x", (1 << 20) + 17);
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("before");
+                d.appendSymbol(big);
+                d.appendSymbol("after");
+                Assert.assertEquals(3, d.size());
+            }
+
+            // Recovery must load ALL three; pre-fix the reopen truncated at the
+            // big entry and came back with size 1 (only "before" survived).
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertEquals("large entry must survive recovery, not be truncated",
+                        3, re.size());
+                ObjList s = re.readLoadedSymbols();
+                Assert.assertEquals("before", s.getQuick(0));
+                Assert.assertEquals(big, s.getQuick(1));
+                Assert.assertEquals("after", s.getQuick(2));
+            }
+        });
+    }
+
+    @Test
+    public void testOpenCleanDiscardsSurvivingDictionary() throws Exception {
+        // A fresh start must NOT inherit a dictionary left by a prior lifecycle:
+        // openClean() truncates any survivor to empty, where open() would recover
+        // (and TRUST) it. Trusting a survivor whose segments are gone -- the
+        // fresh-start producer is not seeded from it -- shifts the dense id->symbol
+        // mapping and misattributes symbols on the next reconnect.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict-clean");
+            try (PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(stale);
+                stale.appendSymbol("staleX");
+                stale.appendSymbol("staleY");
+                Assert.assertEquals(2, stale.size());
+            }
+
+            // Fresh start: openClean yields an EMPTY dictionary regardless of
+            // the survivor, and appends from id 0 again.
+            try (PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString())) {
+                Assert.assertNotNull(fresh);
+                Assert.assertEquals(0, fresh.size());
+                Assert.assertEquals(0, fresh.readLoadedSymbols().size());
+                fresh.appendSymbol("freshA");
+                Assert.assertEquals(1, fresh.size());
+            }
+
+            // The survivor's bytes are physically gone, not just hidden: a
+            // subsequent recovery open() sees only the post-clean content.
+            try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(reopened);
+                Assert.assertEquals(1, reopened.size());
+                Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0));
+            }
+        });
+    }
+
+    @Test
+    public void testRemoveOrphanDeletesFile() throws Exception {
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("A");
+            }
+            Path f = dir.resolve(".symbol-dict");
+            Assert.assertTrue(Files.exists(f));
+            PersistedSymbolDict.removeOrphan(dir.toString());
+            Assert.assertFalse(Files.exists(f));
+        });
+    }
+
+    @Test
+    public void testTooLargeToReopenDegradesWithoutOpeningTheFile() throws Exception {
+        // A dictionary at or past Integer.MAX_VALUE cannot be read into one int-sized
+        // buffer, so open() must short-circuit to null BEFORE touching the file.
+        //
+        // The facade reports 2^32 + 100 -- deliberately NOT Integer.MAX_VALUE + 1.
+        // That value casts to Integer.MIN_VALUE, which Unsafe.malloc rejects anyway,
+        // so openExisting's catch would produce the same null and the test could not
+        // tell the guard from its absence (it was exactly that vacuous before). A
+        // length of 2^32 + k instead casts to a SMALL POSITIVE prefix -- the branch
+        // the guard's own javadoc calls out -- under which openExisting would happily
+        // malloc, read a truncated prefix, and parse it.
+        //
+        // The assertion that pins the guard is therefore NOT the null (both paths give
+        // null) but that the file is never even OPENED: with the guard, open() returns
+        // before openRW; without it, openExisting opens the file and parses garbage.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(seed);
+                for (int i = 0; i < 40; i++) {
+                    seed.appendSymbol("sym" + i);
+                }
+            }
+            Path f = dir.resolve(".symbol-dict");
+            byte[] before = Files.readAllBytes(f);
+
+            HugeLengthFacade ff = new HugeLengthFacade();
+            Assert.assertNull("a >=2GB dictionary must degrade to null",
+                    PersistedSymbolDict.open(ff, dir.toString()));
+            Assert.assertEquals("the guard must short-circuit BEFORE opening the file",
+                    0, ff.openRwCalls);
+            Assert.assertArrayEquals("open() must NOT destroy or trim an oversized dictionary",
+                    before, Files.readAllBytes(f));
+        });
+    }
+
+    @Test
+    public void testTornTrailingEntrySelfHeals() throws Exception {
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            // Write two complete entries, then a torn trailing record: a
+            // length prefix of 5 followed by only 2 bytes (crash mid-append).
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("one");
+                d.appendSymbol("two");
+            }
+
+            Path f = dir.resolve(".symbol-dict");
+            long cleanLen = Files.size(f); // header + "one" + "two", no tail
+            Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'},
+                    StandardOpenOption.APPEND);
+            Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f));
+
+            // Reopen: the torn tail is ignored; only the two complete entries load.
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                // open() physically truncates the torn tail: the file returns
+                // to its clean length, so a later SHORTER append can never
+                // leave residue past its end that a future recovery mis-parses
+                // as a ghost symbol.
+                Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f));
+                Assert.assertEquals(2, re.size());
+                ObjList s = re.readLoadedSymbols();
+                Assert.assertEquals("one", s.getQuick(0));
+                Assert.assertEquals("two", s.getQuick(1));
+                // The next append continues from the truncated tail cleanly.
+                re.appendSymbol("three");
+                Assert.assertEquals(3, re.size());
+            }
+
+            try (PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertEquals(3, re2.size());
+                Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2));
+            }
+        });
+    }
+
+    @Test
+    public void testTransientLengthFaultDegradesWithoutDestroyingTheFile() throws Exception {
+        // open() routes on ff.length(): a value < HEADER_SIZE falls through to the
+        // truncating openFresh(). A genuine sub-header stub reports a length in
+        // [0, HEADER_SIZE), but a TRANSIENT stat failure (an EIO on a flaky disk)
+        // reports the -1 error sentinel for a fully populated file -- and routing that
+        // to openFresh would O_TRUNC the only copy of load-bearing state, the exact
+        // destruction the "Never recreate over an existing file" contract forbids. A
+        // negative length is distinguishable from a stub, so open() must degrade to
+        // null (fall back to full self-sufficient frames) and leave every byte on disk
+        // for a later attempt, once the transient clears.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("one");
+                d.appendSymbol("two");
+            }
+
+            Path f = dir.resolve(".symbol-dict");
+            byte[] before = Files.readAllBytes(f);
+            Assert.assertTrue("a populated dict must exceed the header", before.length > HEADER_SIZE);
+
+            // Reopen through a facade whose length() reports the -1 stat-error
+            // sentinel for the (present) file -- the branch under test.
+            PersistedSymbolDict reopened = PersistedSymbolDict.open(new StatFailsLengthFacade(), dir.toString());
+            if (reopened != null) {
+                // Pre-fix: open() fell through to openFresh() and handed back a fresh
+                // empty dict over the now-truncated file. Close its fd so only the
+                // assertion below reports the failure, not a leaked descriptor.
+                reopened.close();
+            }
+            Assert.assertNull("a populated dict whose length cannot be stat'd must degrade to null, not truncate",
+                    reopened);
+            Assert.assertArrayEquals("a transient length-stat fault must NOT destroy the dictionary",
+                    before, Files.readAllBytes(f));
+
+            // Once the filesystem recovers, the SAME file reopens its intact content.
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(re);
+                Assert.assertEquals("the intact content survives the transient", 2, re.size());
+                ObjList s = re.readLoadedSymbols();
+                Assert.assertEquals("one", s.getQuick(0));
+                Assert.assertEquals("two", s.getQuick(1));
+            }
+        });
+    }
+
+    @Test
+    public void testTruncateFailureDegradesWithoutDestroyingTheFile() throws Exception {
+        // A host crash can leave a torn/stale tail past the last complete chunk.
+        // open() drops it with a truncate; if that truncate FAILS (a read-only
+        // remount, a Windows share lock), the file still exposes the stale bytes,
+        // whose self-consistent chunk CRC a later shifted parse could accept as a real
+        // symbol. So a failed truncate must make the file UNTRUSTED -- open() degrades
+        // to null (the sender falls back to full self-sufficient frames) rather than
+        // returning a dict laid over stale bytes.
+        //
+        // It must NOT recreate the file, which is what it used to do: a read-only
+        // remount is transient, and destroying the [one, two] prefix on the way past it
+        // makes every surviving delta frame permanently unreplayable. Degrading leaves
+        // the bytes for the next attempt, once the mount is writable again.
+        assertMemoryLeak(() -> {
+            Path dir = newFolder("qwp-symdict");
+            try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) {
+                d.appendSymbol("one");
+                d.appendSymbol("two");
+            }
+
+            // Append a torn trailing record so the reopen parses [one, two], then
+            // finds validLen < len and tries to truncate the tail -- the branch
+            // under test.
+            Path f = dir.resolve(".symbol-dict");
+            long cleanLen = Files.size(f); // header + "one" + "two", no tail
+            Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND);
+            Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f));
+            byte[] before = Files.readAllBytes(f);
+
+            // Reopen through a facade whose truncate() fails.
+            Assert.assertNull("a dict whose torn tail cannot be trimmed must degrade to null",
+                    PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString()));
+            Assert.assertArrayEquals("a failed truncate must NOT destroy the dictionary",
+                    before, Files.readAllBytes(f));
+
+            // And once the filesystem is writable again, the SAME file recovers its
+            // intact prefix in full -- which is the whole point of not destroying it.
+            try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) {
+                Assert.assertNotNull(re);
+                Assert.assertEquals("the intact prefix survives the transient", 2, re.size());
+                ObjList s = re.readLoadedSymbols();
+                Assert.assertEquals("one", s.getQuick(0));
+                Assert.assertEquals("two", s.getQuick(1));
+                Assert.assertEquals("the torn tail is trimmed once truncate works", cleanLen, Files.size(f));
+            }
+        });
+    }
+
+    private Path newFolder(String name) throws IOException {
+        return temporaryFolder.newFolder(name).toPath();
+    }
+
+    /**
+     * Fails every {@link #truncate(int, long)} -- reproducing a host where the
+     * torn/stale-tail truncate cannot succeed (read-only remount, Windows share
+     * lock) so {@code open()}'s fail-clean recreate path runs.
+     */
+    private static final class FailingTruncateFacade extends DelegatingFilesFacade {
+        @Override
+        public boolean truncate(int fd, long size) {
+            return false;
+        }
+    }
+
+    /**
+     * Reports a dictionary length past {@link Integer#MAX_VALUE} -- reproducing a
+     * dictionary that legitimately grew beyond 2GB, which {@code open()} cannot read
+     * into one int-sized buffer and must refuse before touching the file.
+     * 

+     * The length is {@code 2^32 + 100}, NOT {@code Integer.MAX_VALUE + 1}. The latter
+     * casts to {@code Integer.MIN_VALUE}, which {@code Unsafe.malloc} rejects on its
+     * own, so {@code openExisting}'s catch would produce the same {@code null} with or
+     * without the guard -- which is precisely what made the old version of this test
+     * vacuous. {@code 2^32 + k} casts to a small POSITIVE prefix instead, so without
+     * the guard {@code openExisting} really would open the file and parse a truncated
+     * prefix of it. {@link #openRwCalls} is what catches that.
+     */
+    private static final class HugeLengthFacade extends DelegatingFilesFacade {
+        int openRwCalls;
+
+        @Override
+        public long length(String path) {
+            return (1L << 32) + 100L;
+        }
+
+        @Override
+        public int openRW(String path) {
+            openRwCalls++;
+            return super.openRW(path);
+        }
+    }
+
+    /**
+     * Lands ONE armed entry append short -- writes {@code len-1} of the {@code len}
+     * requested bytes and reports {@code len-1} -- reproducing a disk-full / quota
+     * short write mid-persist. Fires only on an entry append (offset past the
+     * 8-byte header), never the header write, and disarms after firing so the retry
+     * writes cleanly.
+     */
+    private static final class ShortWriteOnceFacade extends DelegatingFilesFacade {
+        boolean armed;
+
+        @Override
+        public long write(int fd, long addr, long len, long offset) {
+            if (armed && offset > 0 && len > 1) {
+                armed = false;
+                return INSTANCE.write(fd, addr, len - 1, offset);
+            }
+            return INSTANCE.write(fd, addr, len, offset);
+        }
+    }
+
+    /**
+     * Reports a length of -1 -- the stat-error sentinel -- for the dictionary file,
+     * reproducing a transient stat failure (an EIO on a flaky disk) where the file is
+     * present but its size cannot be read. open() must treat this as "present but
+     * unreadable" and degrade to null, NOT route it to the truncating fresh-open path.
+     */
+    private static final class StatFailsLengthFacade extends DelegatingFilesFacade {
+        @Override
+        public long length(String path) {
+            return -1L;
+        }
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java
index 644a7463..d38ef9e5 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java
@@ -103,6 +103,32 @@ public void testCloseReleasesLock() throws Exception {
         });
     }
 
+    @Test
+    public void testLogicalLockRemainsContendedAcrossSlotRenameAndRecreate() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            // Use the platform-native separator. In particular, this exercises
+            // acquireLogical with a backslash-only path on Windows.
+            String slot = Paths.get(parentDir, "rename").toString();
+            String moved = Paths.get(parentDir, "rename.quarantined").toString();
+            assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT));
+
+            try (SlotLock ignored = SlotLock.acquireLogical(slot)) {
+                assertEquals(0, Files.rename(slot, moved));
+                assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT));
+
+                try (SlotLock unexpected = SlotLock.acquireLogical(slot)) {
+                    fail("logical slot lock must survive rename and recreate");
+                } catch (IllegalStateException expected) {
+                    assertTrue(expected.getMessage().contains("already in use"));
+                }
+            }
+
+            try (SlotLock reacquired = SlotLock.acquireLogical(slot)) {
+                assertEquals(slot, reacquired.slotDir());
+            }
+        });
+    }
+
     @Test
     public void testTwoDifferentSlotsCoexist() throws Exception {
         TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java
index 6d4c5ef0..5e009d67 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java
@@ -108,6 +108,11 @@ public class TestWebSocketServer implements Closeable {
     // 401, 403, 404, 426, 503, etc. that the failover loop should
     // classify per failover.md §6.
     private volatile String rejectingStatusReason;
+    // When > 0, 101 upgrade responses advertise this value as the
+    // X-QWP-Max-Batch-Size header, capping the QWP message size the client
+    // builds. Lets a test force the delta-dictionary catch-up to split across
+    // several frames. Live-updatable via setAdvertisedMaxBatchSize().
+    private volatile int advertisedMaxBatchSize;
 
     public TestWebSocketServer(WebSocketServerHandler handler) throws IOException {
         this(handler, false);
@@ -221,6 +226,14 @@ public int liveConnectionCount() {
         return liveConnections.get();
     }
 
+    /**
+     * Advertises {@code X-QWP-Max-Batch-Size: } on subsequent
+     * handshakes (live update). Pass {@code 0} to stop advertising a cap.
+     */
+    public void setAdvertisedMaxBatchSize(int maxBatchSize) {
+        this.advertisedMaxBatchSize = maxBatchSize;
+    }
+
     /**
      * Replaces the advertised role for subsequent handshakes (live update).
      */
@@ -603,6 +616,10 @@ private boolean performHandshake() throws IOException {
             if (role != null) {
                 sb.append("X-QuestDB-Role: ").append(role).append("\r\n");
             }
+            int maxBatch = advertisedMaxBatchSize;
+            if (maxBatch > 0) {
+                sb.append("X-QWP-Max-Batch-Size: ").append(maxBatch).append("\r\n");
+            }
             sb.append("\r\n");
             out.write(sb.toString().getBytes(StandardCharsets.US_ASCII));
             out.flush();
diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
index cfef2feb..2f213c3a 100644
--- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
+++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
@@ -1847,7 +1847,9 @@ private static boolean hasSegmentFile(String slotPath) {
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
                 rc = Files.findNext(find);
-                if (name != null && name.endsWith(".sfa")) {
+                if (name != null
+                        && name.endsWith(".sfa")
+                        && !name.startsWith(".qwp-v2-guard-")) {
                     return true;
                 }
             }
diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java
index dba5765f..f6287825 100644
--- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java
+++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java
@@ -75,6 +75,8 @@ public void testEveryIngressKeyIsHonored() {
         assertHonored("max_background_drainers=6", "max_background_drainers", 6);
         assertHonored("max_frame_rejections=6", "max_frame_rejections", 6);
         assertHonored("poison_min_escalation_window_millis=7500", "poison_min_escalation_window_millis", 7500L);
+        assertHonored("catchup_cap_gap_min_escalation_window_millis=90000",
+                "catchup_cap_gap_min_escalation_window_millis", 90000L);
         assertHonored("error_inbox_capacity=128", "error_inbox_capacity", 128);
         assertHonored("connection_listener_inbox_capacity=64", "connection_listener_inbox_capacity", 64);
         assertHonored("token=ey.abc", "token", "ey.abc");
diff --git a/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java
new file mode 100644
index 00000000..516800c4
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java
@@ -0,0 +1,170 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.tools;
+
+import io.questdb.client.std.FilesFacade;
+
+/**
+ * Pass-through {@link FilesFacade} that forwards every call to
+ * {@link FilesFacade#INSTANCE}. Subclass it and override the ONE method whose
+ * failure you want to inject -- a short write, a truncate that cannot trim, an
+ * implausible file length -- so a test can drive a real I/O failure path without a
+ * real broken disk.
+ * 

+ * {@code FilesFacade} has no default methods, so the delegation boilerplate is
+ * unavoidable; it lives here once rather than being stamped into each test that
+ * needs a fault seam.
+ */
+public class DelegatingFilesFacade implements FilesFacade {
+    @Override
+    public long allocNativePath(String path) {
+        return INSTANCE.allocNativePath(path);
+    }
+
+    @Override
+    public boolean allocate(int fd, long size) {
+        return INSTANCE.allocate(fd, size);
+    }
+
+    @Override
+    public int close(int fd) {
+        return INSTANCE.close(fd);
+    }
+
+    @Override
+    public boolean exists(String path) {
+        return INSTANCE.exists(path);
+    }
+
+    @Override
+    public void findClose(long findPtr) {
+        INSTANCE.findClose(findPtr);
+    }
+
+    @Override
+    public long findFirst(String dir) {
+        return INSTANCE.findFirst(dir);
+    }
+
+    @Override
+    public long findName(long findPtr) {
+        return INSTANCE.findName(findPtr);
+    }
+
+    @Override
+    public int findNext(long findPtr) {
+        return INSTANCE.findNext(findPtr);
+    }
+
+    @Override
+    public int findType(long findPtr) {
+        return INSTANCE.findType(findPtr);
+    }
+
+    @Override
+    public void freeNativePath(long pathPtr) {
+        INSTANCE.freeNativePath(pathPtr);
+    }
+
+    @Override
+    public int fsync(int fd) {
+        return INSTANCE.fsync(fd);
+    }
+
+    @Override
+    public long length(int fd) {
+        return INSTANCE.length(fd);
+    }
+
+    @Override
+    public long length(String path) {
+        return INSTANCE.length(path);
+    }
+
+    @Override
+    public long length(long pathPtr) {
+        return INSTANCE.length(pathPtr);
+    }
+
+    @Override
+    public int lock(int fd) {
+        return INSTANCE.lock(fd);
+    }
+
+    @Override
+    public int mkdir(String path, int mode) {
+        return INSTANCE.mkdir(path, mode);
+    }
+
+    @Override
+    public int openCleanRW(String path) {
+        return INSTANCE.openCleanRW(path);
+    }
+
+    @Override
+    public int openCleanRW(long pathPtr) {
+        return INSTANCE.openCleanRW(pathPtr);
+    }
+
+    @Override
+    public int openRW(String path) {
+        return INSTANCE.openRW(path);
+    }
+
+    @Override
+    public int openRW(long pathPtr) {
+        return INSTANCE.openRW(pathPtr);
+    }
+
+    @Override
+    public long read(int fd, long addr, long len, long offset) {
+        return INSTANCE.read(fd, addr, len, offset);
+    }
+
+    @Override
+    public boolean remove(String path) {
+        return INSTANCE.remove(path);
+    }
+
+    @Override
+    public boolean remove(long pathPtr) {
+        return INSTANCE.remove(pathPtr);
+    }
+
+    @Override
+    public int rename(String oldPath, String newPath) {
+        return INSTANCE.rename(oldPath, newPath);
+    }
+
+    @Override
+    public boolean truncate(int fd, long size) {
+        return INSTANCE.truncate(fd, size);
+    }
+
+    @Override
+    public long write(int fd, long addr, long len, long offset) {
+        return INSTANCE.write(fd, addr, len, offset);
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
index 7572880b..82f65e6b 100644
--- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
+++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
@@ -37,9 +37,15 @@
 import org.junit.Assert;
 import org.slf4j.Logger;
 
+import java.io.IOException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.file.FileVisitResult;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Arrays;
 
 import static org.junit.Assert.assertNotNull;
@@ -215,6 +221,91 @@ public static void removeTmpDir(String tmpDir) {
         Files.remove(tmpDir);
     }
 
+    /**
+     * Recursive counterpart to {@link #removeTmpDir(String)} for tests whose temp
+     * directory has subdirectories -- e.g. the store-and-forward slot layout
+     * {@code 
/default/...}, which the flat variant cannot clean up. A
+     * {@code null} argument is a no-op, so it is safe from {@code tearDown} before
+     * {@code setUp} ran. Uses {@code java.nio.file} (fully qualified to avoid the
+     * {@code io.questdb.client.std.Files} import clash) so subdirectories delete
+     * bottom-up.
+     */
+    public static void removeTmpDirRec(String tmpDir) {
+        if (tmpDir == null) {
+            return;
+        }
+        removeTmpDirRec(Paths.get(tmpDir), java.nio.file.Files::deleteIfExists);
+    }
+
+    /**
+     * Injectable counterpart used to verify that cleanup remains best-effort but
+     * still fails the test when any path cannot be removed.
+     */
+    static void removeTmpDirRec(Path root, PathDeleter deleter) {
+        final IOException[] firstFailure = new IOException[1];
+        try {
+            java.nio.file.Files.walkFileTree(root, new SimpleFileVisitor() {
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+                    remember(exc);
+                    delete(dir);
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                    delete(file);
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+                    remember(exc);
+                    return FileVisitResult.CONTINUE;
+                }
+
+                private void delete(Path path) {
+                    try {
+                        deleter.delete(path);
+                    } catch (IOException e) {
+                        remember(e);
+                    }
+                }
+
+                private void remember(IOException failure) {
+                    if (failure == null || failure instanceof NoSuchFileException) {
+                        // The cleanup goal for this path is already satisfied. This
+                        // also covers a child disappearing between directory listing
+                        // and attribute lookup.
+                        return;
+                    }
+                    if (firstFailure[0] == null) {
+                        firstFailure[0] = failure;
+                    } else if (firstFailure[0] != failure) {
+                        firstFailure[0].addSuppressed(failure);
+                    }
+                }
+            });
+        } catch (NoSuchFileException ignored) {
+            // A missing root is an already-complete cleanup.
+        } catch (IOException e) {
+            if (firstFailure[0] == null) {
+                firstFailure[0] = e;
+            } else {
+                firstFailure[0].addSuppressed(e);
+            }
+        }
+        if (firstFailure[0] != null) {
+            throw new AssertionError("could not recursively remove temp directory: " + root,
+                    firstFailure[0]);
+        }
+    }
+
+    @FunctionalInterface
+    interface PathDeleter {
+        void delete(Path path) throws IOException;
+    }
+
     /**
      * Java 8 stand-in for {@code String.repeat(int)} (added in Java 11).
      */
diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java b/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java
new file mode 100644
index 00000000..61cee092
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java
@@ -0,0 +1,65 @@
+/*+*****************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.tools;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class TestUtilsTest {
+
+    @Test
+    public void testRecursiveCleanupRetainsFirstFailureAndContinuesDeleting() throws Exception {
+        Path root = Files.createTempDirectory("qdb-test-utils-cleanup-");
+        Path child = Files.createDirectory(root.resolve("child"));
+        Path file = Files.createFile(child.resolve("blocked"));
+        IOException injected = new IOException("simulated deletion failure");
+        AtomicInteger attempts = new AtomicInteger();
+        try {
+            try {
+                TestUtils.removeTmpDirRec(root, path -> {
+                    attempts.incrementAndGet();
+                    if (path.equals(file)) {
+                        throw injected;
+                    }
+                    Files.deleteIfExists(path);
+                });
+                Assert.fail("cleanup must report the injected deletion failure");
+            } catch (AssertionError e) {
+                Assert.assertSame("the first deletion failure must be retained", injected, e.getCause());
+                Assert.assertTrue("cleanup must continue with parent directories after a file failure",
+                        attempts.get() >= 3);
+                Assert.assertTrue("later parent deletion failures must remain diagnostic",
+                        e.getCause().getSuppressed().length >= 2);
+            }
+        } finally {
+            TestUtils.removeTmpDirRec(root.toString());
+        }
+    }
+}