From 3cd7d2bb5d0e3bc520305f7e1872c29dc68ef305 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Wed, 14 Oct 2026 19:58:40 +0200 Subject: [PATCH] feat(gooddata-eval): select Langfuse dataset items by id or metadata A Langfuse dataset is the unit of experiment comparison: Langfuse only compares runs within one dataset. That makes it the right place to keep every variant of a question -- its locales, for instance -- and the wrong place to split them, since splitting puts the variants on charts that can never be compared. But `load_langfuse_dataset` had no filter, so the only way to run one locale was a dispatch of the whole dataset, and `DatasetItem` was `extra="ignore"` with no metadata field, so the item metadata that says which locale an item is was discarded on load. You could not filter on what had already been dropped. So: - `DatasetItem.metadata` carries the source's own item metadata through verbatim. `test_kind`, `summary_input` and `user_context` are still read out into their own fields by the loader; this is the rest of it. - `load_langfuse_dataset(..., item_ids=, metadata_filter=)` narrows after the fetch -- the dataset-items endpoint selects by dataset but not by item id or metadata. The fetch costs the same either way; what the filters save is evaluating an item against a live agent. Given both, an item must satisfy both. - An id the dataset does not have raises `DatasetItemNotFoundError` naming what is missing, rather than returning the ids that did exist: asking for twelve items and evaluating nine, silently, reads downstream as a pass-rate change. - `--langfuse-item-id` and `--langfuse-item-metadata KEY=VALUE` expose both, repeatable. With a local dataset folder they are refused, not ignored, for the same reason. - `--run-metadata-json PATH` reaches the existing `run_metadata_extra` path, which was plumbed end-to-end into the dataset-run metadata but had no way in from the CLI. Per run, not per item. A non-object payload is rejected up front, since run metadata merges key by key and a list would be dropped after the run had already cost its agent calls. No behaviour change for a caller using none of them: an unfiltered load passes `None` rather than an empty container, which would otherwise select zero items and raise. Verified against the live `mastercard_search_tool` dataset (48 items, 12 per locale): unfiltered 48, `language=en` 12, `language=es` 12, a single id 1, an unknown id raises. Co-Authored-By: Claude Opus 5 --- packages/gooddata-eval/README.md | 10 ++ .../src/gooddata_eval/cli/agentic_runner.py | 10 +- .../src/gooddata_eval/cli/main.py | 76 ++++++++++- .../src/gooddata_eval/core/config.py | 8 +- .../core/dataset/langfuse_source.py | 70 ++++++++++- .../src/gooddata_eval/core/models.py | 7 ++ .../tests/test_agentic_runner.py | 30 +++++ .../tests/test_cli_langfuse_flags.py | 115 +++++++++++++++++ .../tests/test_langfuse_source.py | 118 ++++++++++++++++++ 9 files changed, 436 insertions(+), 8 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_cli_langfuse_flags.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index d6b614955..8e76c2906 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -117,8 +117,17 @@ gd-eval run \ |---|---| | `--dataset PATH` | Flat folder of JSON files — one question per file. | | `--langfuse-dataset NAME` | Pull items by name from a Langfuse dataset. Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` and `LANGFUSE_BASE_URL` (or the legacy `LANGFUSE_HOST`). | +| `--langfuse-item-id ITEM_ID` | Run only this item from the Langfuse dataset. Repeatable. An id the dataset does not have is an error, not a skip — asking for twelve items and evaluating nine, silently, reads downstream as a pass-rate change. Requires `--langfuse-dataset`. | +| `--langfuse-item-metadata KEY=VALUE` | Run only the items whose Langfuse item metadata carries this key and value, e.g. `--langfuse-item-metadata language=es`. Repeatable; several are ANDed, and values compare as strings. Combined with `--langfuse-item-id`, an item must satisfy both. Requires `--langfuse-dataset`. | | `--kind TEST_KIND` | Fallback `test_kind` for dataset items that do not embed one. Defaults to `visualization`; use e.g. `agentic_metric_skill` for multi-turn agentic evaluation. Items that declare their own `test_kind` ignore this. | +Both item filters are applied after the fetch — the Langfuse dataset-items endpoint selects +by dataset but not by item id or item metadata. The fetch costs the same either way; what +the filters save is the part that actually costs, evaluating an item against a live agent. +They exist so one published dataset can hold every variant of a question (its locales, say) +and still be dispatched a slice at a time, instead of being split into datasets that +Langfuse can then no longer compare against each other. + #### Model selection | Flag | Description | @@ -159,6 +168,7 @@ interleaves when K > 1, and per-item latencies rise, so they stop being clean si | Flag | Description | |---|---| | `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Names each experiment run `{dataset_name}_{timestamp}_{model}`, suffixed `_effort-{level}` when `--reasoning-effort` is set (so runs differing only by effort stay separate) and `_run{N}` per run when `--runs` > 1 — e.g. `general_question_2026-09-02-11-13_gpt-5.2_run0`. Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` and `LANGFUSE_BASE_URL` (or the legacy `LANGFUSE_HOST`). | +| `--run-metadata-json PATH` | A JSON object merged into the Langfuse **dataset-run** metadata. Per run, not per item — for what is constant across the run and otherwise only recoverable from the repo that launched it: the agent id and its personality text, the workspace actually evaluated, the memory snapshot in force. A payload that is not an object is rejected up front, since run metadata merges key by key and a list would be dropped after the run had already cost its agent calls. Merged under `--tag`, which wins on a `tags` key. | ##### Langfuse v4 diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 09d0dc267..382b5a222 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -5,6 +5,7 @@ import sys import time +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, TypedDict @@ -405,6 +406,7 @@ def run_agentic_items( concurrency: int = 1, gate: EvalGate = DEFAULT_GATE, tags: tuple[str, ...] = (), + run_metadata: Mapping[str, Any] | None = None, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport. @@ -417,10 +419,16 @@ def run_agentic_items( automatic ones. They reach Langfuse whenever trace linking runs, with or without ``use_langfuse``. + ``run_metadata`` is merged into the Langfuse dataset-run metadata, once per run rather + than once per item. ``tags`` wins on a "tags" key, since that one is derived here. + Results are collected in dataset order regardless of completion order. """ langfuse = make_langfuse_client() if use_langfuse else None - run_metadata_extra: dict[str, Any] | None = {"tags": list(tags)} if tags else None + extra: dict[str, Any] = dict(run_metadata or {}) + if tags: + extra["tags"] = list(tags) + run_metadata_extra: dict[str, Any] | None = extra or None report = EvalReport(model=model_version, gate=normalize_gate(gate)) total = len(items) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 8d90b12ce..636493a4c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -2,12 +2,13 @@ """`gd-eval` command-line entry point.""" import argparse +import json import os import sys import threading from datetime import datetime, timezone from pathlib import Path -from typing import get_args +from typing import Any, get_args import httpx from gooddata_api_client.exceptions import ApiException, ApiTypeError @@ -78,6 +79,35 @@ def _build_parser() -> argparse.ArgumentParser: source = run.add_mutually_exclusive_group(required=True) source.add_argument("--dataset", help="Path to a folder of dataset JSON files.") source.add_argument("--langfuse-dataset", dest="langfuse_dataset", help="Langfuse dataset name.") + run.add_argument( + "--langfuse-item-id", + dest="langfuse_item_ids", + action="append", + default=[], + metavar="ITEM_ID", + help="Run only this Langfuse dataset item. Repeat for several. An id the dataset does " + "not have is an error, not a skip. Requires --langfuse-dataset.", + ) + run.add_argument( + "--langfuse-item-metadata", + dest="langfuse_item_metadata", + action="append", + default=[], + type=_parse_item_metadata, + metavar="KEY=VALUE", + help="Run only Langfuse items whose metadata carries this key and value (e.g. " + "--langfuse-item-metadata language=es). Repeat to require several; values compare as " + "strings. Combined with --langfuse-item-id, an item must satisfy both. " + "Requires --langfuse-dataset.", + ) + run.add_argument( + "--run-metadata-json", + dest="run_metadata_json", + metavar="PATH", + help="JSON object merged into the Langfuse dataset-run metadata. Per run, not per item " + "-- use it for what is constant across the run (agent id, personality, workspace, " + "memory snapshot id), not for anything that varies by question.", + ) run.add_argument( "--kind", dest="kind", @@ -396,6 +426,30 @@ def _parse_tag(val: str) -> str: return tag +def _parse_item_metadata(val: str) -> tuple[str, str]: + """Split one --langfuse-item-metadata KEY=VALUE pair.""" + key, sep, value = val.partition("=") + if not sep or not key.strip(): + raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got {val!r}") + return key.strip(), value.strip() + + +def _load_run_metadata(path: str) -> dict[str, Any]: + """Read --run-metadata-json. + + A non-object payload is rejected here rather than at the Langfuse call: dataset-run + metadata is merged key by key, so a list or a bare string has nowhere to go and would + otherwise be dropped silently after the run had already cost its agent calls. + """ + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + raise ValueError(f"--run-metadata-json {path}: not valid JSON ({e})") from e + if not isinstance(payload, dict): + raise ValueError(f"--run-metadata-json {path}: expected a JSON object, got {type(payload).__name__}") + return payload + + def _make_progress_callbacks(console: Console): """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress. @@ -446,7 +500,12 @@ def _load_dataset(config: RunConfig): if config.langfuse_dataset is None: # pragma: no cover - argparse mutually-exclusive group guarantees one is set raise ValueError("Either --dataset or --langfuse-dataset is required.") - return load_langfuse_dataset(config.langfuse_dataset, default_test_kind=config.kind) + return load_langfuse_dataset( + config.langfuse_dataset, + default_test_kind=config.kind, + item_ids=config.item_ids or None, + metadata_filter=config.item_metadata_filter or None, + ) def _list_models(host: str, token: str, workspace_id: str | None) -> int: @@ -510,6 +569,15 @@ def _run(config: RunConfig) -> int: file=sys.stderr, ) return _EXIT_OPERATIONAL_ERROR + if (config.item_ids or config.item_metadata_filter) and config.langfuse_dataset is None: + # Refused rather than ignored: a run that quietly evaluated the whole local folder + # after being asked for one language would be read as that language's pass rate. + print( + "error: --langfuse-item-id / --langfuse-item-metadata require --langfuse-dataset " + "(a local dataset folder has no item metadata to select on).", + file=sys.stderr, + ) + return _EXIT_OPERATIONAL_ERROR items = _load_dataset(config) agentic_items = [i for i in items if i.test_kind in AGENTIC_TEST_KINDS] @@ -595,6 +663,7 @@ def on_langfuse_item_done( reasoning_effort=config.reasoning_effort, use_langfuse=config.log_to_langfuse, tags=config.tags, + run_metadata=config.run_metadata, run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, @@ -750,6 +819,9 @@ def main(argv: list[str] | None = None) -> int: turn_timeout_s=args.turn_timeout, item_timeout_s=args.item_timeout, tags=tuple(dict.fromkeys(args.tags)), + item_ids=tuple(dict.fromkeys(args.langfuse_item_ids)), + item_metadata_filter=dict(args.langfuse_item_metadata), + run_metadata=_load_run_metadata(args.run_metadata_json) if args.run_metadata_json else {}, ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 4d9032831..98d7b8b56 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -4,7 +4,7 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import Literal, cast, get_args +from typing import Any, Literal, cast, get_args # Values that mean "off" when read from the environment. Needed because every non-empty # string is truthy in Python, so a bare bool() on an env var reads FOO=0 -- the obvious way @@ -99,3 +99,9 @@ class RunConfig: gate: EvalGate = DEFAULT_GATE tags: tuple[str, ...] = () """Custom Langfuse trace tags, added after the automatic ones.""" + item_ids: tuple[str, ...] = () + """Langfuse dataset items to run, by id. Empty runs them all. Ignored for a local dataset.""" + item_metadata_filter: dict[str, str] = field(default_factory=dict) + """Run only Langfuse items whose metadata carries all these key/values. Ignored for a local dataset.""" + run_metadata: dict[str, Any] = field(default_factory=dict) + """Extra key/values written into the Langfuse dataset-run metadata, per run, not per item.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py index 415cc77f2..8c4e91c24 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py @@ -12,6 +12,7 @@ LANGFUSE_HOST — base URL, legacy alias for LANGFUSE_BASE_URL """ +from collections.abc import Collection, Mapping from typing import Any, TypeVar, cast import httpx @@ -24,6 +25,19 @@ _T = TypeVar("_T") +class DatasetItemNotFoundError(LookupError): + """Named item ids were asked for and the dataset does not have them. + + Raised rather than silently returning the ids that did exist: asking for twelve items + and evaluating nine, with nothing said, reads downstream as a pass-rate change. + """ + + def __init__(self, dataset_name: str, missing: Collection[str]) -> None: + self.dataset_name = dataset_name + self.missing = sorted(missing) + super().__init__(f"Langfuse dataset {dataset_name!r} has no item(s) with id: {', '.join(self.missing)}") + + def _make_client() -> httpx.Client: """Build an httpx client with Langfuse basic-auth headers.""" return make_http_client(timeout=30) @@ -124,21 +138,52 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt expected_output=expected_output, summary_input=_summary_input_from_raw(raw, expected_output), user_context=_user_context_from_raw(raw), + metadata=metadata if isinstance(metadata := raw.get("metadata"), dict) else {}, ) -def load_langfuse_dataset(name: str, *, default_test_kind: str = "visualization") -> list[DatasetItem]: - """Pull all items from a Langfuse dataset by name via the REST API. +def _matches_metadata(item: DatasetItem, metadata_filter: Mapping[str, str]) -> bool: + """True when the item's metadata carries every wanted key with the wanted value. + + Values are compared as strings because the filter arrives from a CLI `KEY=VALUE` pair, + where everything is a string, while the item side may hold a number or a bool. A key + the item does not have never matches -- an item with no `language` is not an item in + every language. + """ + return all(key in item.metadata and str(item.metadata[key]) == value for key, value in metadata_filter.items()) + + +def load_langfuse_dataset( + name: str, + *, + default_test_kind: str = "visualization", + item_ids: Collection[str] | None = None, + metadata_filter: Mapping[str, str] | None = None, +) -> list[DatasetItem]: + """Pull items from a Langfuse dataset by name via the REST API. + + Both filters are applied client-side, after the fetch: the Langfuse dataset-items + endpoint can select by dataset but not by item id or item metadata, so narrowing has + to happen here. The fetch cost is the whole dataset either way; what the filters save + is the part that actually costs -- evaluating an item against a live agent. Args: name: The Langfuse dataset name (as shown in the Langfuse UI). default_test_kind: Fallback test_kind when the item doesn't specify one. + item_ids: Keep only these item ids. Every id must exist, or it raises -- see + `DatasetItemNotFoundError`. + metadata_filter: Keep only items whose metadata carries all of these key/value + pairs, compared as strings. + + Given both, an item has to satisfy both. Returns: - Parsed dataset items. + Parsed dataset items, in the order Langfuse returned them. Raises: RuntimeError: Missing Langfuse credentials or dataset not found. + ValueError: The dataset is empty, or the filters selected nothing from it. + DatasetItemNotFoundError: `item_ids` named an id the dataset does not have. """ items: list[dict] = [] page = 1 @@ -165,4 +210,21 @@ def load_langfuse_dataset(name: str, *, default_test_kind: str = "visualization" if not items: raise ValueError(f"Langfuse dataset '{name}' exists but contains no items.") - return [_item_from_raw(raw, dataset_name=name, test_kind=default_test_kind) for raw in items] + parsed = [_item_from_raw(raw, dataset_name=name, test_kind=default_test_kind) for raw in items] + + if item_ids is not None: + wanted = set(item_ids) + if missing := wanted - {item.id for item in parsed}: + raise DatasetItemNotFoundError(name, missing) + parsed = [item for item in parsed if item.id in wanted] + + if metadata_filter: + parsed = [item for item in parsed if _matches_metadata(item, metadata_filter)] + + if not parsed: + raise ValueError( + f"Langfuse dataset '{name}' has items, but none matched the requested filter " + f"(metadata_filter={dict(metadata_filter or {})!r})." + ) + + return parsed diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 9b2b1c4a7..080c6aa58 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -477,3 +477,10 @@ class DatasetItem(BaseModel): # that schema (a discriminated union of view/widget descriptors), so re-modelling it here # would only create a second copy to keep in sync. user_context: dict[str, Any] | None = None + # The source's own item metadata, carried through verbatim. A Langfuse dataset item can + # hold whatever its publisher put there -- language, complexity, workspace id, the git + # revision the fixture was published from -- and `extra="ignore"` above used to drop all + # of it on load, so a caller could neither select on it nor report it. Not interpreted + # here: `test_kind`, `summary_input` and `user_context` are still read out into their own + # fields by the loader, and this is the rest of it. + metadata: dict[str, Any] = Field(default_factory=dict) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 3ccd72230..4507b6c55 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -984,6 +984,36 @@ def test_run_agentic_items_passes_custom_tags_as_run_metadata(tags, expected_ext assert mock_eval.call_args.kwargs["run_metadata_extra"] == expected_extra +@pytest.mark.parametrize( + ("run_metadata", "tags", "expected_extra"), + [ + # --run-metadata-json alone: reaches the dataset-run metadata with no tags key. + ({"agent_id": "a-1", "child_workspace_id": "ws-1"}, (), {"agent_id": "a-1", "child_workspace_id": "ws-1"}), + # Both: merged, not one replacing the other. + ({"agent_id": "a-1"}, ("nightly",), {"agent_id": "a-1", "tags": ["nightly"]}), + # A "tags" key in the JSON file loses -- tags is derived here from --tag. + ({"tags": ["from-json"]}, ("nightly",), {"tags": ["nightly"]}), + # Neither: unchanged from before this argument existed. + ({}, (), None), + ], +) +def test_run_agentic_items_merges_run_metadata_with_tags(run_metadata, tags, expected_extra): + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + return_value=AgenticEvalOutcome(detail={"alert_created": True}), + ) as mock_eval: + run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + tags=tags, + run_metadata=run_metadata, + ) + assert mock_eval.call_args.kwargs["run_metadata_extra"] == expected_extra + + class _FirstSend(BaseException): """Ends a run at its first chat turn; BaseException so no kind's error handling keeps it.""" diff --git a/packages/gooddata-eval/tests/test_cli_langfuse_flags.py b/packages/gooddata-eval/tests/test_cli_langfuse_flags.py new file mode 100644 index 000000000..4418d6542 --- /dev/null +++ b/packages/gooddata-eval/tests/test_cli_langfuse_flags.py @@ -0,0 +1,115 @@ +# (C) 2026 GoodData Corporation +"""--langfuse-item-id / --langfuse-item-metadata / --run-metadata-json. + +The three flags that let a caller run part of a Langfuse dataset and say what the run was, +plus a regression guard that a run using none of them is dispatched exactly as before. +""" + +import json +from unittest.mock import patch + +import pytest +from gooddata_eval.cli import main as cli_main +from gooddata_eval.core.config import RunConfig + +_BASE = ["run", "--host", "h", "--workspace", "w", "--langfuse-dataset", "mastercard_search_tool"] + + +def test_item_id_is_repeatable(): + args = cli_main.parse_args([*_BASE, "--langfuse-item-id", "q1-en", "--langfuse-item-id", "q2-en"]) + assert args.langfuse_item_ids == ["q1-en", "q2-en"] + + +def test_item_metadata_parses_key_value_pairs(): + args = cli_main.parse_args( + [*_BASE, "--langfuse-item-metadata", "language=es", "--langfuse-item-metadata", "complexity=Basic"] + ) + assert args.langfuse_item_metadata == [("language", "es"), ("complexity", "Basic")] + + +@pytest.mark.parametrize("bad", ["language", "=es", " =es"]) +def test_item_metadata_rejects_a_pair_without_a_key(bad): + with pytest.raises(SystemExit): + cli_main.parse_args([*_BASE, "--langfuse-item-metadata", bad]) + + +def test_item_metadata_keeps_an_empty_value(): + """`key=` is a real filter -- an item whose metadata holds "" for that key.""" + args = cli_main.parse_args([*_BASE, "--langfuse-item-metadata", "language="]) + assert args.langfuse_item_metadata == [("language", "")] + + +def test_all_three_flags_default_to_empty(): + args = cli_main.parse_args([*_BASE]) + assert args.langfuse_item_ids == [] + assert args.langfuse_item_metadata == [] + assert args.run_metadata_json is None + + +def test_run_metadata_json_is_read_from_the_file(tmp_path): + path = tmp_path / "run.json" + path.write_text(json.dumps({"agent_id": "a-1", "child_workspace_id": "ws-1"}), encoding="utf-8") + assert cli_main._load_run_metadata(str(path)) == {"agent_id": "a-1", "child_workspace_id": "ws-1"} + + +@pytest.mark.parametrize("payload", ["[1, 2]", '"a string"', "42"]) +def test_run_metadata_json_rejects_a_non_object(tmp_path, payload): + """Dataset-run metadata is merged key by key, so a list has nowhere to go.""" + path = tmp_path / "run.json" + path.write_text(payload, encoding="utf-8") + with pytest.raises(ValueError, match="expected a JSON object"): + cli_main._load_run_metadata(str(path)) + + +def test_run_metadata_json_rejects_malformed_json(tmp_path): + path = tmp_path / "run.json" + path.write_text("{not json", encoding="utf-8") + with pytest.raises(ValueError, match="not valid JSON"): + cli_main._load_run_metadata(str(path)) + + +def test_item_filters_reach_load_langfuse_dataset(): + config = RunConfig( + host="h", + token="t", + workspace_id="w", + langfuse_dataset="mastercard_search_tool", + kind="search_tool", + item_ids=("q1-en",), + item_metadata_filter={"language": "en"}, + ) + with patch("gooddata_eval.core.dataset.langfuse_source.load_langfuse_dataset") as load: + cli_main._load_dataset(config) + load.assert_called_once_with( + "mastercard_search_tool", + default_test_kind="search_tool", + item_ids=("q1-en",), + metadata_filter={"language": "en"}, + ) + + +def test_without_the_flags_the_loader_is_called_as_before(): + """Regression guard: an unfiltered Langfuse run must pass None, not an empty container. + + An empty tuple reaching `item_ids` would select zero items and raise, turning every + existing unfiltered run into an error. + """ + config = RunConfig(host="h", token="t", workspace_id="w", langfuse_dataset="ds", kind="search_tool") + with patch("gooddata_eval.core.dataset.langfuse_source.load_langfuse_dataset") as load: + cli_main._load_dataset(config) + load.assert_called_once_with("ds", default_test_kind="search_tool", item_ids=None, metadata_filter=None) + + +@pytest.mark.parametrize( + "extra", + [{"item_ids": ("q1-en",)}, {"item_metadata_filter": {"language": "es"}}], +) +def test_item_filters_with_a_local_dataset_are_refused(tmp_path, capsys, extra): + """Ignoring them would run the whole local folder and report it as that language.""" + config = RunConfig(host="h", token="t", workspace_id="w", dataset_folder=tmp_path, **extra) + assert cli_main._run(config) != 0 + assert "require --langfuse-dataset" in capsys.readouterr().err + + +# run_metadata's journey into the dataset-run metadata is covered behaviourally in +# test_agentic_runner.py, next to the --tag case it merges with. diff --git a/packages/gooddata-eval/tests/test_langfuse_source.py b/packages/gooddata-eval/tests/test_langfuse_source.py index 92448b01e..4b3daddb0 100644 --- a/packages/gooddata-eval/tests/test_langfuse_source.py +++ b/packages/gooddata-eval/tests/test_langfuse_source.py @@ -3,6 +3,7 @@ import pytest from gooddata_eval.core.dataset.langfuse_source import ( + DatasetItemNotFoundError, _infer_test_kind, _item_from_raw, _make_client, @@ -213,6 +214,123 @@ def test_a_blank_test_kind_declaration_falls_back_to_the_default(): assert _infer_test_kind({"test_kind": " agentic_guardrail "}, "visualization") == "agentic_guardrail" +_EXPECTED = {"visualization": {"id": "x", "type": "", "query": {"fields": {}}}} + +# One question, four locales -- the shape every published fixture dataset actually has. +_LOCALE_ITEMS = [ + {**_raw_item(f"q1-{lang}", "Show revenue", _EXPECTED), "metadata": {"language": lang, "complexity": "Basic"}} + for lang in ("en", "es", "fr", "pt") +] + + +@pytest.fixture +def locale_dataset(monkeypatch): + """Serve `_LOCALE_ITEMS` from a mocked REST API, and hand back the mock client.""" + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"data": _LOCALE_ITEMS, "meta": {"totalItems": len(_LOCALE_ITEMS)}} + + mock_client = MagicMock() + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=False) + mock_client.get.return_value = mock_resp + + with patch("gooddata_eval.core.dataset.langfuse_source.httpx.Client", return_value=mock_client): + yield mock_client + + +def test_item_metadata_survives_the_load(locale_dataset): + """DatasetItem used to be `extra="ignore"` with no metadata field, so this was dropped.""" + items = load_langfuse_dataset("mastercard_search_tool") + assert items[0].metadata == {"language": "en", "complexity": "Basic"} + + +def test_an_item_without_metadata_gets_an_empty_dict(monkeypatch): + item = _item_from_raw(_raw_item("lf-nometa", "Show revenue", _EXPECTED), dataset_name="ds", test_kind="vis_agentic") + assert item.metadata == {} + + +def test_no_filter_returns_every_item(locale_dataset): + assert len(load_langfuse_dataset("mastercard_search_tool")) == 4 + + +def test_metadata_filter_selects_one_locale(locale_dataset): + items = load_langfuse_dataset("mastercard_search_tool", metadata_filter={"language": "es"}) + assert [item.id for item in items] == ["q1-es"] + + +def test_metadata_filter_requires_every_key_to_match(locale_dataset): + """Keys are ANDed; a dataset-wide key plus a selective one still narrows.""" + items = load_langfuse_dataset("mastercard_search_tool", metadata_filter={"language": "fr", "complexity": "Basic"}) + assert [item.id for item in items] == ["q1-fr"] + + +def test_metadata_filter_matching_nothing_raises_rather_than_running_zero_items(locale_dataset): + with pytest.raises(ValueError, match="none matched the requested filter"): + load_langfuse_dataset("mastercard_search_tool", metadata_filter={"language": "de"}) + + +def test_item_ids_selects_a_subset_and_keeps_source_order(locale_dataset): + items = load_langfuse_dataset("mastercard_search_tool", item_ids=["q1-pt", "q1-en"]) + assert [item.id for item in items] == ["q1-en", "q1-pt"] + + +def test_item_ids_and_metadata_filter_intersect(locale_dataset): + items = load_langfuse_dataset( + "mastercard_search_tool", item_ids=["q1-en", "q1-es"], metadata_filter={"language": "es"} + ) + assert [item.id for item in items] == ["q1-es"] + + +def test_an_unknown_item_id_raises_and_names_what_is_missing(locale_dataset): + """Silently returning the ids that did exist would read downstream as a pass-rate change.""" + with pytest.raises(DatasetItemNotFoundError) as excinfo: + load_langfuse_dataset("mastercard_search_tool", item_ids=["q1-en", "q1-de", "q1-zz"]) + assert excinfo.value.missing == ["q1-de", "q1-zz"] + assert "q1-de, q1-zz" in str(excinfo.value) + + +def test_a_filter_key_the_item_does_not_have_never_matches(locale_dataset): + """An item with no `runner` is not an item for every runner.""" + with pytest.raises(ValueError, match="none matched the requested filter"): + load_langfuse_dataset("mastercard_search_tool", metadata_filter={"runner": "1"}) + + +def test_metadata_values_compare_as_strings(monkeypatch): + """The filter arrives from a CLI `KEY=VALUE`, so the item's own int/bool must still match.""" + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + + raw = [{**_raw_item("q-num", "Show revenue", _EXPECTED), "metadata": {"runner": 1, "critical": False}}] + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"data": raw, "meta": {"totalItems": 1}} + mock_client = MagicMock() + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=False) + mock_client.get.return_value = mock_resp + + with patch("gooddata_eval.core.dataset.langfuse_source.httpx.Client", return_value=mock_client): + items = load_langfuse_dataset("ds", metadata_filter={"runner": "1", "critical": "False"}) + assert [item.id for item in items] == ["q-num"] + + +def test_filtering_does_not_change_the_request_sent_to_langfuse(locale_dataset): + """The endpoint cannot select by item id or metadata; filtering is ours, after the fetch. + + Guards against a later "optimisation" that passes these through as query params, which + Langfuse would ignore -- the filter would silently stop working. + """ + load_langfuse_dataset("mastercard_search_tool", item_ids=["q1-en"], metadata_filter={"language": "en"}) + params = locale_dataset.get.call_args[1]["params"] + assert set(params) == {"datasetName", "limit", "page"} + + def test_a_blank_declaration_does_not_hide_a_real_one_behind_it(): # The lookup must not stop at the first *string* it finds: a blank # expectedOutput.test_kind used to shadow a valid metadata.test_kind, and the item