diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/html_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/html_report.py index 10db5cec8..b943cb369 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/html_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/html_report.py @@ -74,6 +74,20 @@ def _model_of(label: str, run: dict) -> str: return run.get("model") or (label.split(" · ", 1)[1] if " · " in label else label) +def _alias_name(n: int) -> str: + """``Model A`` .. ``Model Z``, then ``Model AA``, ``Model AB``, ... + + Spelled out rather than ``chr(65 + n)``, which walks straight off the end of the alphabet: the + 27th model came out as ``Model [``, then ``Model \\``, ``Model ]``. + """ + letters = "" + n += 1 + while n: + n, remainder = divmod(n - 1, 26) + letters = chr(65 + remainder) + letters + return f"Model {letters}" + + def _aliases(doc: dict) -> dict[str, tuple[str, str]]: """Map each run label to its redacted ``(run label, model name)``. @@ -92,7 +106,7 @@ def _aliases(doc: dict) -> dict[str, tuple[str, str]]: } letters: dict[tuple[str, str], str] = {} for model in identity.values(): - letters.setdefault(model, f"Model {chr(65 + len(letters))}") + letters.setdefault(model, _alias_name(len(letters))) per_model = Counter(identity.values()) seen: Counter[tuple[str, str]] = Counter() aliases: dict[str, tuple[str, str]] = {} diff --git a/packages/gooddata-eval/tests/test_html_report.py b/packages/gooddata-eval/tests/test_html_report.py index f78b99303..d90413cdd 100644 --- a/packages/gooddata-eval/tests/test_html_report.py +++ b/packages/gooddata-eval/tests/test_html_report.py @@ -7,7 +7,7 @@ import orjson import pytest from gooddata_eval.cli.main import main -from gooddata_eval.core.reporting.html_report import build_html, load_report_files, merge_docs +from gooddata_eval.core.reporting.html_report import _aliases, build_html, load_report_files, merge_docs def _doc(model: str, passed: bool) -> dict: @@ -129,6 +129,17 @@ def test_redact_gives_one_model_one_alias_across_its_runs() -> None: assert {entry["provider_name"] for entry in data["comparison"].values()} == {""} +def test_redact_alias_survives_past_twenty_six_models() -> None: + """chr(65 + n) walked off the alphabet: the 27th model was `Model [`.""" + doc = {"runs": {f"m{i}": {"model": f"model-{i}", "items": {}} for i in range(30)}, "comparison": {}} + + names = [name.removeprefix("Model ") for _, name in _aliases(doc).values()] + + assert len(set(names)) == 30 + assert all(name.isalpha() for name in names), names + assert names[25:28] == ["Z", "AA", "AB"] + + def test_redact_gives_each_model_its_own_letter_and_numbers_only_a_repeated_model() -> None: merged = merge_docs([("a", _doc("gpt-5", True)), ("b", _doc("claude", True)), ("c", _doc("gpt-5", False))])