fix(net): report a JS IETF FETCH group evicted after listing as unknown

js/net/src/ietf/fetch.test.ts

@@ -595,6 +595,49 @@ test("a timed-out group with nothing yet keeps the groups before it nonexistent"
 	}
 });
 
+// A group the cache evicts between listing it for the answer and writing it is unknown, not
+// skipped, which would say it never existed. Waiting on the open Largest Object's group, before
+// FETCH_OK, gives the older groups time to age out of the cache; the newer one stays.
+test("a group evicted after it was listed is unknown", async () => {
+	for (const version of VERSIONS) {
+		for (const order of ["ascending", "descending"] as const) {
+			using clock = manual();
+			using fx = serve(version);
+			// Group 0 never existed; groups 1 and 2 age out.
+			writeGroup(fx.track, 1, ["1-0", "1-1"]);
+			writeGroup(fx.track, 2, ["2-0", "2-1"]);
+			await clock.advance(ms(25_000));
+			writeGroup(fx.track, 3, ["3-0", "3-1"]);
+			const live = writeGroup(fx.track, 4, ["4-0"], false);
+
+			const type = range(version, at(0, 0), at(4, 0));
+			const msg = new Fetch({ requestId: FETCH_ID, type, groupOrder: order, fillTimeout: Milli(20_000) });
+			const answer = served(await clock.drive(fetch(fx, msg)));
+			expect(answer.ok.endLocation).toEqual({ group: 4n, object: version === D18 ? 0n : MAX_OBJECT });
+
+			const newest: [number, number, string][] = [[4, 0, "4-0"], gap(4, MAX_OBJECT, timedOut(version))];
+			expect(answer.objects).toEqual(
+				order === "ascending"
+					? [
+							gap(0, MAX_OBJECT, "8c"),
+							gap(1, MAX_OBJECT, "10c"),
+							gap(2, MAX_OBJECT, "10c"),
+							...pairs([3]),
+							...newest,
+						]
+					: [
+							...newest,
+							...pairs([3]),
+							gap(3, MAX_OBJECT, "8c"),
+							gap(2, MAX_OBJECT, "10c"),
+							gap(1, MAX_OBJECT, "10c"),
+						],
+			);
+			live.finish();
+		}
+	}
+});
+
 // Every FETCH keeps its own budget, even for the same open group, and one without FILL_TIMEOUT,
 // as an older peer sends, waits for the group as it always has.
 test("each FETCH keeps its own FILL_TIMEOUT", async () => {

js/net/src/ietf/publisher.test.ts

@@ -3330,6 +3330,56 @@ test("draft-22: a fill's FILL_TIMEOUT marks what it did not wait for", async ()
 	}
 });
 
+// A group the cache evicts after the fill listed it is unknown, not skipped, which would say it
+// never existed. Waiting on the open group the fill reaches first gives the older groups time
+// to age out of the cache.
+test("draft-22: a fill marks a group evicted after it was listed unknown", async () => {
+	using clock = manual();
+	const fx = fixture();
+	const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI });
+	// Groups 0 and 1 age out; the open group 2 is the newest the fill covers.
+	writeGroup(track, 2);
+	writeGroup(track, 2);
+	await clock.advance(Nano.fromMilli(Milli(25_000)));
+	const open = track.appendGroup();
+	open.writeFrame({ payload: new TextEncoder().encode("2.0"), timestamp: Timestamp.now() });
+	writeGroup(track, 1);
+
+	const { client } = await runSubscribe(
+		fx,
+		new Subscribe({
+			requestId: 7n,
+			trackNamespace: Path.from("test"),
+			trackName: "video",
+			subscriberPriority: 0,
+			filter: { kind: "nextObject" },
+			fill: {
+				filter: { kind: "absolute", startGroup: 0n, startObject: 0n, endGroup: 2n },
+				fillTimeout: Milli(20_000),
+				rangeFilters: false,
+			},
+		}),
+	);
+
+	try {
+		const fill = await nextUni(fx.uni);
+		if (!fill) throw new Error("no fetch stream for the requested fill");
+		const served = await clock.drive(readFill(fill));
+		expect(served.reset).toBeUndefined();
+		const end = Number(2n ** 62n - 1n);
+		expect(served.objects).toEqual([
+			{ group: 2, id: 0, payload: "2.0" },
+			{ group: 2, id: end, payload: "end 20c" },
+			{ group: 1, id: end, payload: "end 10c" },
+			{ group: 0, id: end, payload: "end 10c" },
+		]);
+	} finally {
+		open.finish();
+		fx.close();
+		client.close();
+	}
+});
+
 /**
  * A broadcast served through `requested()` resolves its track on demand, and a dynamic serve
  * is deliberately one request per peer subscription. Resolving the track again to read the

js/net/src/ietf/publisher.ts

@@ -1460,9 +1460,13 @@ export class Publisher {
 						: { group: fill.end.group, object: 0n },
 			});
 			for (const sequence of sequences) {
-				const group = hooks.cachedGroup(cache, sequence);
-				if (!group) continue;
 				const slice = groupSlice(fill, sequence);
+				const group = hooks.cachedGroup(cache, sequence);
+				// Evicted since it was listed: unknown, where a skip would say it never existed.
+				if (!group) {
+					await out.unknown(sequence, slice, cancelled);
+					continue;
+				}
 				try {
 					await this.#writeFillGroup({ out, group, slice, deadline, cancelled });
 				} finally {
@@ -3087,8 +3091,9 @@ export class Publisher {
 				{ start: { included: safe(range.start.group) }, end: { included: safe(range.end.group) } },
 				priority,
 			);
-			// A group gone or never cached is absent from the answer, as the response skips it,
-			// unless it is the only one.
+			// The answer ends at the newest group still readable, so one gone from the top of the
+			// range moves FETCH_OK's end below it rather than leaving it a gap, unless it is the
+			// only one.
 			let last: FetchedGroup | undefined;
 			while (!last) {
 				const sequence = groups.pop();
@@ -3317,11 +3322,14 @@ export class Publisher {
 					if (last.timedOut) await out.timedOut(sequence, fetchSlice(range, sequence), stop);
 					continue;
 				}
-				// Gone since it was listed: absent from the answer, which says it does not exist.
+				const slice = fetchSlice(range, sequence);
 				const group = hooks.cachedGroup(cache, sequence);
-				if (!group) continue;
+				// Evicted since it was listed: unknown, where a skip would say it never existed.
+				if (!group) {
+					await out.unknown(sequence, slice, stop);
+					continue;
+				}
 				try {
-					const slice = fetchSlice(range, sequence);
 					await this.#writeFillGroup({ out, group, slice, deadline, cancelled: stop });
 				} finally {
 					group.close();
@@ -3667,19 +3675,34 @@ class FetchWriter {
 	 * Mark the rest of group `sequence`'s `slice` as timed out, its objects having not
 	 * arrived before the FILL_TIMEOUT ran out. Draft-18 has no such marker; its FILL_TIMEOUT
 	 * reports them unknown.
+	 */
+	timedOut(sequence: number, slice: GroupSlice, stop: AbortSignal): Promise<void> {
+		const reason = Filter.isDraft22(this.#version) ? EndOfRange.TimedOut : EndOfRange.Unknown;
+		return this.#gap(reason, sequence, slice, stop);
+	}
+
+	/**
+	 * Mark group `sequence`'s whole `slice` as unknown: the cache evicted the group after it
+	 * was listed, so whether its objects exist is no longer known here.
+	 */
+	unknown(sequence: number, slice: GroupSlice, stop: AbortSignal): Promise<void> {
+		return this.#gap(EndOfRange.Unknown, sequence, slice, stop);
+	}
+
+	/**
+	 * End the rest of group `sequence`'s `slice` with an End of Range for `reason`.
 	 *
 	 * A marker covers everything since the prior one, so when nothing of this group went
 	 * out, the groups skipped before it are first marked as what they otherwise imply: gone.
 	 */
-	async timedOut(sequence: number, slice: GroupSlice, stop: AbortSignal): Promise<void> {
+	async #gap(reason: EndOfRange, sequence: number, slice: GroupSlice, stop: AbortSignal): Promise<void> {
 		const group = BigInt(sequence);
 		const prior = this.#prior;
 		if (prior?.group !== group) {
 			const before = this.#before({ group, object: BigInt(slice.skip) });
 			if (before && (prior ? this.#precedes(prior, before) : !this.#precedes(before, this.#start)))
 				await this.#end(EndOfRange.NonExistent, before, stop);
 		}
-		const reason = Filter.isDraft22(this.#version) ? EndOfRange.TimedOut : EndOfRange.Unknown;
 		const object = slice.until === undefined ? MAX_OBJECT : BigInt(slice.until - 1);
 		await this.#end(reason, { group, object }, stop);
 	}

quest/m1/README.md

@@ -156,7 +156,6 @@ removed).
 - [A moq-transport request token is one auth per distinct token](/quest/m1/ietf-token-auth.md) - requests naming the same token share one verification and grant, on any request type, ending with the last such request
 - [moq-quic's API improves on quinn's](/quest/m1/quic-api/README.md) - moq-quic and both QUIC drivers ship one mirrored API: `send::`/`recv::` namespaces, one error per direction, consuming terminal operations, plain configs, no quinn baggage
 - [On-demand JavaScript FETCH](/quest/m1/js-fetch-on-demand.md) - browser publishers answer IETF FETCH for evicted or uncached groups through the JS ranges request surface
-- [JS evicted FETCH groups are unknown](/quest/m1/js-fetch-evicted-unknown.md) - a group evicted between listing and writing goes out as End of Unknown Range on drafts 18 and 22, not a silent skip
 - [Rust FETCH parity](/quest/m1/ietf-fetch-parity.md) - Rust finishes an answered joining FETCH's head, refuses a lost head with DOES_NOT_EXIST, and sends no timestamps without TIMESCALE, as the JS publisher does
 - [SUBSCRIBE honors the properties opt-out](/quest/m1/subscribe-timestamps.md) - the Rust publisher sends untimed objects to a SUBSCRIBE that declines Track Properties, and the Rust subscriber asks for them when it needs timestamps
 - [JS FETCH timeout test](/quest/m1/js-fetch-timeout-test.md) - a JS joining FETCH still waiting on its SUBSCRIBE at the deadline is refused with TIMEOUT

quest/m1/ietf-publisher-one-group.md

@@ -75,9 +75,9 @@ Implementation:
 - **JS publisher.** Same rules in `#answerFetch` and `#runFill`. Remove the
   FILL_TIMEOUT deadline and its timed-out markers from
   [moq-dev/moq#7051](https://github.com/moq-dev/moq/pull/7051). Keep the End
-  of Range encoding, which
-  [JS evicted FETCH groups are unknown](/quest/m1/js-fetch-evicted-unknown.md)
-  uses for a group lost after FETCH_OK.
+  of Range encoding, which `FetchWriter.unknown()` from
+  [moq-dev/moq#7150](https://github.com/moq-dev/moq/pull/7150) uses for a
+  group evicted after it was listed.
 - **Tests.** Unit tests in both publishers for each refusal and for each
   kept form on drafts 18 and 22. `warm_backfill` in
   `rs/moq-net/tests/integration` keeps draft 22 out of its backfill and