From eb129c3ebed3643e0b5190c2cc2984dd36dcfdd6 Mon Sep 17 00:00:00 2001 From: Achord Chan Date: Thu, 3 Sep 2026 18:49:55 +0800 Subject: [PATCH 1/2] fix(benchmarks): apply limit after shuffling --- CHANGELOG.md | 5 + benchmarks/public/runner/run_subprocess.py | 1 - tests/test_benchmark_runner_sampling.py | 155 +++++++++++++++++++++ 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/test_benchmark_runner_sampling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3aee46f..a4ae801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,3 +33,8 @@ Initial open-source release of FrontierAgent. inside the configured context window. - Clean-machine Linux + NVIDIA installation and release-certification guide, distinguishing deployment health from production agent correctness. + +### Fixed + +- Apply benchmark question limits after seeded shuffling so repeated runs can + sample different questions while `--no-shuffle` keeps canonical ordering. diff --git a/benchmarks/public/runner/run_subprocess.py b/benchmarks/public/runner/run_subprocess.py index a578413..0c0ae60 100644 --- a/benchmarks/public/runner/run_subprocess.py +++ b/benchmarks/public/runner/run_subprocess.py @@ -266,7 +266,6 @@ async def run_eval( questions = load_questions( args.benchmark, - limit=args.limit, offset=args.offset, answer_type=args.answer_type, category=args.category, diff --git a/tests/test_benchmark_runner_sampling.py b/tests/test_benchmark_runner_sampling.py new file mode 100644 index 0000000..8b39b50 --- /dev/null +++ b/tests/test_benchmark_runner_sampling.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import argparse +from types import SimpleNamespace + +import pytest + +from benchmarks.public.core.question import BenchmarkQuestion + + +class _SelectionCaptured(Exception): + """Stop a benchmark run after its selected questions are observable.""" + + +def _questions(count: int = 20) -> list[BenchmarkQuestion]: + return [ + BenchmarkQuestion( + id=f"q{index:02d}", + question=f"Question {index}", + ground_truth=f"Answer {index}", + answer_type="exactMatch", + ) + for index in range(count) + ] + + +@pytest.mark.asyncio +async def test_runner_applies_limit_after_seeded_shuffle( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + from benchmarks.public import sandbox_profiles + from benchmarks.public.core import harbor_task_generator, registry + from benchmarks.public.runner import run_subprocess + + source = _questions() + selected_runs: list[list[str]] = [] + load_calls: list[dict[str, object]] = [] + + monkeypatch.setattr( + registry, + "get_config", + lambda _benchmark: SimpleNamespace( + default_pipeline="stateful-react-agent", + scoring_mode="external", + name="Sample", + ), + ) + + def load_questions(_benchmark: str, **kwargs: object) -> list[BenchmarkQuestion]: + load_calls.append(kwargs) + return source.copy() + + monkeypatch.setattr(registry, "load_questions", load_questions) + monkeypatch.setattr( + sandbox_profiles, + "resolve_closed_book", + lambda _benchmark, _override=None: False, + ) + + def capture_selection(question_dicts, _tasks_dir, *, pipeline_id: str) -> None: + assert pipeline_id == "stateful-react-agent" + selected_runs.append([question["id"] for question in question_dicts]) + raise _SelectionCaptured + + monkeypatch.setattr( + harbor_task_generator, + "generate_task_dirs", + capture_selection, + ) + + args = argparse.Namespace( + benchmark="sample", + pipeline=None, + web=None, + limit=5, + offset=2, + answer_type=None, + category=None, + no_shuffle=False, + profile="default", + fs_mode=False, + ) + + for seed in (42, 1234): + with pytest.raises(_SelectionCaptured): + await run_subprocess.run_eval( + args, + out_dir=tmp_path / str(seed), + seed=seed, + ) + + assert all("limit" not in call for call in load_calls) + assert len(selected_runs[0]) == len(selected_runs[1]) == args.limit + assert set(selected_runs[0]) != set(selected_runs[1]) + + +@pytest.mark.asyncio +async def test_runner_limit_preserves_order_when_shuffle_is_disabled( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + from benchmarks.public import sandbox_profiles + from benchmarks.public.core import harbor_task_generator, registry + from benchmarks.public.runner import run_subprocess + + source = _questions() + selected: list[str] = [] + + monkeypatch.setattr( + registry, + "get_config", + lambda _benchmark: SimpleNamespace( + default_pipeline="stateful-react-agent", + scoring_mode="external", + name="Sample", + ), + ) + monkeypatch.setattr( + registry, + "load_questions", + lambda _benchmark, **_kwargs: source.copy(), + ) + monkeypatch.setattr( + sandbox_profiles, + "resolve_closed_book", + lambda _benchmark, _override=None: False, + ) + + def capture_selection(question_dicts, _tasks_dir, *, pipeline_id: str) -> None: + assert pipeline_id == "stateful-react-agent" + selected.extend(question["id"] for question in question_dicts) + raise _SelectionCaptured + + monkeypatch.setattr( + harbor_task_generator, + "generate_task_dirs", + capture_selection, + ) + + args = argparse.Namespace( + benchmark="sample", + pipeline=None, + web=None, + limit=5, + offset=2, + answer_type=None, + category=None, + no_shuffle=True, + profile="default", + fs_mode=False, + ) + + with pytest.raises(_SelectionCaptured): + await run_subprocess.run_eval(args, out_dir=tmp_path, seed=42) + + assert selected == [question.id for question in source[: args.limit]] From 2cff817f7c813003c7e9a0af8e9ecb61a73713cd Mon Sep 17 00:00:00 2001 From: Achord Chan Date: Tue, 8 Sep 2026 14:31:19 +0800 Subject: [PATCH 2/2] test(benchmarks): parameterize sampling regression coverage --- tests/test_benchmark_runner_sampling.py | 101 ++++++------------------ 1 file changed, 23 insertions(+), 78 deletions(-) diff --git a/tests/test_benchmark_runner_sampling.py b/tests/test_benchmark_runner_sampling.py index 8b39b50..e226263 100644 --- a/tests/test_benchmark_runner_sampling.py +++ b/tests/test_benchmark_runner_sampling.py @@ -25,16 +25,28 @@ def _questions(count: int = 20) -> list[BenchmarkQuestion]: @pytest.mark.asyncio -async def test_runner_applies_limit_after_seeded_shuffle( - tmp_path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + ("seed", "no_shuffle", "expected_ids"), + [ + (42, False, ["q19", "q05", "q14", "q04", "q09"]), + (1234, False, ["q19", "q13", "q04", "q09", "q16"]), + (42, True, ["q00", "q01", "q02", "q03", "q04"]), + ], + ids=["seed-42", "seed-1234", "no-shuffle"], +) +async def test_runner_selects_questions_after_optional_shuffle( + tmp_path, + monkeypatch: pytest.MonkeyPatch, + seed: int, + no_shuffle: bool, + expected_ids: list[str], ) -> None: from benchmarks.public import sandbox_profiles from benchmarks.public.core import harbor_task_generator, registry from benchmarks.public.runner import run_subprocess source = _questions() - selected_runs: list[list[str]] = [] - load_calls: list[dict[str, object]] = [] + selected: list[str] = [] monkeypatch.setattr( registry, @@ -46,9 +58,10 @@ async def test_runner_applies_limit_after_seeded_shuffle( ), ) - def load_questions(_benchmark: str, **kwargs: object) -> list[BenchmarkQuestion]: - load_calls.append(kwargs) - return source.copy() + def load_questions( + _benchmark: str, *, limit: int | None = None, **_kwargs: object + ) -> list[BenchmarkQuestion]: + return source[:limit] if limit else source.copy() monkeypatch.setattr(registry, "load_questions", load_questions) monkeypatch.setattr( @@ -57,74 +70,6 @@ def load_questions(_benchmark: str, **kwargs: object) -> list[BenchmarkQuestion] lambda _benchmark, _override=None: False, ) - def capture_selection(question_dicts, _tasks_dir, *, pipeline_id: str) -> None: - assert pipeline_id == "stateful-react-agent" - selected_runs.append([question["id"] for question in question_dicts]) - raise _SelectionCaptured - - monkeypatch.setattr( - harbor_task_generator, - "generate_task_dirs", - capture_selection, - ) - - args = argparse.Namespace( - benchmark="sample", - pipeline=None, - web=None, - limit=5, - offset=2, - answer_type=None, - category=None, - no_shuffle=False, - profile="default", - fs_mode=False, - ) - - for seed in (42, 1234): - with pytest.raises(_SelectionCaptured): - await run_subprocess.run_eval( - args, - out_dir=tmp_path / str(seed), - seed=seed, - ) - - assert all("limit" not in call for call in load_calls) - assert len(selected_runs[0]) == len(selected_runs[1]) == args.limit - assert set(selected_runs[0]) != set(selected_runs[1]) - - -@pytest.mark.asyncio -async def test_runner_limit_preserves_order_when_shuffle_is_disabled( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: - from benchmarks.public import sandbox_profiles - from benchmarks.public.core import harbor_task_generator, registry - from benchmarks.public.runner import run_subprocess - - source = _questions() - selected: list[str] = [] - - monkeypatch.setattr( - registry, - "get_config", - lambda _benchmark: SimpleNamespace( - default_pipeline="stateful-react-agent", - scoring_mode="external", - name="Sample", - ), - ) - monkeypatch.setattr( - registry, - "load_questions", - lambda _benchmark, **_kwargs: source.copy(), - ) - monkeypatch.setattr( - sandbox_profiles, - "resolve_closed_book", - lambda _benchmark, _override=None: False, - ) - def capture_selection(question_dicts, _tasks_dir, *, pipeline_id: str) -> None: assert pipeline_id == "stateful-react-agent" selected.extend(question["id"] for question in question_dicts) @@ -144,12 +89,12 @@ def capture_selection(question_dicts, _tasks_dir, *, pipeline_id: str) -> None: offset=2, answer_type=None, category=None, - no_shuffle=True, + no_shuffle=no_shuffle, profile="default", fs_mode=False, ) with pytest.raises(_SelectionCaptured): - await run_subprocess.run_eval(args, out_dir=tmp_path, seed=42) + await run_subprocess.run_eval(args, out_dir=tmp_path, seed=seed) - assert selected == [question.id for question in source[: args.limit]] + assert selected == expected_ids