From 497f63869c5e12ebbaa63e10c5877eef486e2c03 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 15:04:44 -0700 Subject: [PATCH 1/2] chore(quest): claim quest/m1/js-fetch-evicted-unknown.md Quest-Claim: 5a62fc18faf9e7fa From ae4cbc7ee7ee745d279344259ebb389ed3f41ce5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 15:18:18 -0700 Subject: [PATCH 2/2] fix(net): report a FETCH or fill group evicted after listing as unknown A group the cache evicts between listing it for a JS IETF FETCH answer or fill and writing it now goes out as End of Unknown Range (0x10C) on drafts 18 and 22, instead of a silent skip that reads as the group not existing. Co-Authored-By: Claude Opus 5.5 --- js/net/src/ietf/fetch.test.ts | 43 ++++++++++++++++++++++++ js/net/src/ietf/publisher.test.ts | 50 ++++++++++++++++++++++++++++ js/net/src/ietf/publisher.ts | 41 ++++++++++++++++++----- quest/m1/README.md | 1 - quest/m1/js-fetch-evicted-unknown.md | 34 ------------------- 5 files changed, 125 insertions(+), 44 deletions(-) delete mode 100644 quest/m1/js-fetch-evicted-unknown.md diff --git a/js/net/src/ietf/fetch.test.ts b/js/net/src/ietf/fetch.test.ts index 49483a43678..633a9e3c20e 100644 --- a/js/net/src/ietf/fetch.test.ts +++ b/js/net/src/ietf/fetch.test.ts @@ -595,6 +595,49 @@ test("a timed-out group with nothing yet keeps the groups before it nonexistent" } }); +// A group the cache evicts between listing it for the answer and writing it is unknown, not +// skipped, which would say it never existed. Waiting on the open Largest Object's group, before +// FETCH_OK, gives the older groups time to age out of the cache; the newer one stays. +test("a group evicted after it was listed is unknown", async () => { + for (const version of VERSIONS) { + for (const order of ["ascending", "descending"] as const) { + using clock = manual(); + using fx = serve(version); + // Group 0 never existed; groups 1 and 2 age out. + writeGroup(fx.track, 1, ["1-0", "1-1"]); + writeGroup(fx.track, 2, ["2-0", "2-1"]); + await clock.advance(ms(25_000)); + writeGroup(fx.track, 3, ["3-0", "3-1"]); + const live = writeGroup(fx.track, 4, ["4-0"], false); + + const type = range(version, at(0, 0), at(4, 0)); + const msg = new Fetch({ requestId: FETCH_ID, type, groupOrder: order, fillTimeout: Milli(20_000) }); + const answer = served(await clock.drive(fetch(fx, msg))); + expect(answer.ok.endLocation).toEqual({ group: 4n, object: version === D18 ? 0n : MAX_OBJECT }); + + const newest: [number, number, string][] = [[4, 0, "4-0"], gap(4, MAX_OBJECT, timedOut(version))]; + expect(answer.objects).toEqual( + order === "ascending" + ? [ + gap(0, MAX_OBJECT, "8c"), + gap(1, MAX_OBJECT, "10c"), + gap(2, MAX_OBJECT, "10c"), + ...pairs([3]), + ...newest, + ] + : [ + ...newest, + ...pairs([3]), + gap(3, MAX_OBJECT, "8c"), + gap(2, MAX_OBJECT, "10c"), + gap(1, MAX_OBJECT, "10c"), + ], + ); + live.finish(); + } + } +}); + // Every FETCH keeps its own budget, even for the same open group, and one without FILL_TIMEOUT, // as an older peer sends, waits for the group as it always has. test("each FETCH keeps its own FILL_TIMEOUT", async () => { diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index 690ed2a3c03..786690355ca 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -3341,6 +3341,56 @@ test("draft-22: a fill's FILL_TIMEOUT marks what it did not wait for", async () } }); +// A group the cache evicts after the fill listed it is unknown, not skipped, which would say it +// never existed. Waiting on the open group the fill reaches first gives the older groups time +// to age out of the cache. +test("draft-22: a fill marks a group evicted after it was listed unknown", async () => { + using clock = manual(); + const fx = fixture(); + const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI }); + // Groups 0 and 1 age out; the open group 2 is the newest the fill covers. + writeGroup(track, 2); + writeGroup(track, 2); + await clock.advance(Nano.fromMilli(Milli(25_000))); + const open = track.appendGroup(); + open.writeFrame({ payload: new TextEncoder().encode("2.0"), timestamp: Timestamp.now() }); + writeGroup(track, 1); + + const { client } = await runSubscribe( + fx, + new Subscribe({ + requestId: 7n, + trackNamespace: Path.from("test"), + trackName: "video", + subscriberPriority: 0, + filter: { kind: "nextObject" }, + fill: { + filter: { kind: "absolute", startGroup: 0n, startObject: 0n, endGroup: 2n }, + fillTimeout: Milli(20_000), + rangeFilters: false, + }, + }), + ); + + try { + const fill = await nextUni(fx.uni); + if (!fill) throw new Error("no fetch stream for the requested fill"); + const served = await clock.drive(readFill(fill)); + expect(served.reset).toBeUndefined(); + const end = Number(2n ** 62n - 1n); + expect(served.objects).toEqual([ + { group: 2, id: 0, payload: "2.0" }, + { group: 2, id: end, payload: "end 20c" }, + { group: 1, id: end, payload: "end 10c" }, + { group: 0, id: end, payload: "end 10c" }, + ]); + } finally { + open.finish(); + fx.close(); + client.close(); + } +}); + /** * A broadcast served through `requested()` resolves its track on demand, and a dynamic serve * is deliberately one request per peer subscription. Resolving the track again to read the diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index 77cd37cccc0..c55c8169860 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -1460,9 +1460,13 @@ export class Publisher { : { group: fill.end.group, object: 0n }, }); for (const sequence of sequences) { - const group = hooks.cachedGroup(cache, sequence); - if (!group) continue; const slice = groupSlice(fill, sequence); + const group = hooks.cachedGroup(cache, sequence); + // Evicted since it was listed: unknown, where a skip would say it never existed. + if (!group) { + await out.unknown(sequence, slice, cancelled); + continue; + } try { await this.#writeFillGroup({ out, group, slice, deadline, cancelled }); } finally { @@ -3087,8 +3091,9 @@ export class Publisher { { start: { included: safe(range.start.group) }, end: { included: safe(range.end.group) } }, priority, ); - // A group gone or never cached is absent from the answer, as the response skips it, - // unless it is the only one. + // The answer ends at the newest group still readable, so one gone from the top of the + // range moves FETCH_OK's end below it rather than leaving it a gap, unless it is the + // only one. let last: FetchedGroup | undefined; while (!last) { const sequence = groups.pop(); @@ -3317,11 +3322,14 @@ export class Publisher { if (last.timedOut) await out.timedOut(sequence, fetchSlice(range, sequence), stop); continue; } - // Gone since it was listed: absent from the answer, which says it does not exist. + const slice = fetchSlice(range, sequence); const group = hooks.cachedGroup(cache, sequence); - if (!group) continue; + // Evicted since it was listed: unknown, where a skip would say it never existed. + if (!group) { + await out.unknown(sequence, slice, stop); + continue; + } try { - const slice = fetchSlice(range, sequence); await this.#writeFillGroup({ out, group, slice, deadline, cancelled: stop }); } finally { group.close(); @@ -3667,11 +3675,27 @@ class FetchWriter { * Mark the rest of group `sequence`'s `slice` as timed out, its objects having not * arrived before the FILL_TIMEOUT ran out. Draft-18 has no such marker; its FILL_TIMEOUT * reports them unknown. + */ + timedOut(sequence: number, slice: GroupSlice, stop: AbortSignal): Promise { + const reason = Filter.isDraft22(this.#version) ? EndOfRange.TimedOut : EndOfRange.Unknown; + return this.#gap(reason, sequence, slice, stop); + } + + /** + * Mark group `sequence`'s whole `slice` as unknown: the cache evicted the group after it + * was listed, so whether its objects exist is no longer known here. + */ + unknown(sequence: number, slice: GroupSlice, stop: AbortSignal): Promise { + return this.#gap(EndOfRange.Unknown, sequence, slice, stop); + } + + /** + * End the rest of group `sequence`'s `slice` with an End of Range for `reason`. * * A marker covers everything since the prior one, so when nothing of this group went * out, the groups skipped before it are first marked as what they otherwise imply: gone. */ - async timedOut(sequence: number, slice: GroupSlice, stop: AbortSignal): Promise { + async #gap(reason: EndOfRange, sequence: number, slice: GroupSlice, stop: AbortSignal): Promise { const group = BigInt(sequence); const prior = this.#prior; if (prior?.group !== group) { @@ -3679,7 +3703,6 @@ class FetchWriter { if (before && (prior ? this.#precedes(prior, before) : !this.#precedes(before, this.#start))) await this.#end(EndOfRange.NonExistent, before, stop); } - const reason = Filter.isDraft22(this.#version) ? EndOfRange.TimedOut : EndOfRange.Unknown; const object = slice.until === undefined ? MAX_OBJECT : BigInt(slice.until - 1); await this.#end(reason, { group, object }, stop); } diff --git a/quest/m1/README.md b/quest/m1/README.md index 32a58c043cd..8a6ee2f6211 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -149,7 +149,6 @@ removed). - [moq-quic's API improves on quinn's](/quest/m1/quic-api/README.md) - moq-quic and both QUIC drivers ship one mirrored API: `send::`/`recv::` namespaces, one error per direction, consuming terminal operations, plain configs, no quinn baggage - [Drop web-transport-trait](/quest/m1/drop-web-transport-trait.md) - iroh, wasm, and ffi implement moq-net's `transport::poll` traits, and the remaining `web_transport_trait` adapters are deleted - [On-demand JavaScript FETCH](/quest/m1/js-fetch-on-demand.md) - browser publishers answer IETF FETCH for evicted or uncached groups through the JS ranges request surface -- [JS evicted FETCH groups are unknown](/quest/m1/js-fetch-evicted-unknown.md) - a group evicted between listing and writing goes out as End of Unknown Range on drafts 18 and 22, not a silent skip - [Rust FETCH parity](/quest/m1/ietf-fetch-parity.md) - Rust finishes an answered joining FETCH's head, refuses a lost head with DOES_NOT_EXIST, and sends no timestamps without TIMESCALE, as the JS publisher does - [SUBSCRIBE honors the properties opt-out](/quest/m1/subscribe-timestamps.md) - the Rust publisher sends untimed objects to a SUBSCRIBE that declines Track Properties, and the Rust subscriber asks for them when it needs timestamps - [JS FETCH timeout test](/quest/m1/js-fetch-timeout-test.md) - a JS joining FETCH still waiting on its SUBSCRIBE at the deadline is refused with TIMEOUT diff --git a/quest/m1/js-fetch-evicted-unknown.md b/quest/m1/js-fetch-evicted-unknown.md deleted file mode 100644 index babe540bc97..00000000000 --- a/quest/m1/js-fetch-evicted-unknown.md +++ /dev/null @@ -1,34 +0,0 @@ -# [S] JS IETF FETCH reports an evicted group as unknown - -## Goal - -When the cache evicts a group after a JS IETF FETCH or fill has listed it but -before writing it, the publisher sends End of Unknown Range (`0x10C`) for that -group on drafts 18 and 22, instead of silently skipping it, which reads as -the group not existing. - -## Plan - -Found while landing [moq-dev/moq#7051](https://github.com/moq-dev/moq/pull/7051), -which sends Unknown or Timed-Out markers when FILL_TIMEOUT runs out. The silent -skips are the "Gone since it was listed" path in `#answerFetch` and the -`!group` path in `#runFill` (`js/net/src/ietf/publisher.ts`). - -Decided with the maintainer on 2026-10-28: - -- **`0x10C` on both drafts.** Both define End of Unknown Range, and the - FILL_TIMEOUT path already sends it on draft 18. Nothing ends the FETCH - early, and FETCH_OK's end is unchanged. -- **In m1**, so both JS gap paths are honest at the cut. - -Test on mocked time: evict a listed group between listing and writing, on -both drafts, for a FETCH and for a fill, and check the requester sees an -Unknown range rather than a skip. Check whether Rust's IETF publisher has the -same skip; if so, fix it here too or note it in the PR. - -Public API: none. Wire: an evicted group is reported as unknown instead of -being left out. - -## Related - -- [On-demand JavaScript FETCH](/quest/m1/js-fetch-on-demand.md) - serves evicted groups when an application responder can; this covers the case where nothing can