From 220532d8961e6c738fe24accfafae5eecf6de4f2 Mon Sep 17 00:00:00 2001
From: huyenthanh09 <huyen.nguyen@gooddata.com>
Date: Thu, 22 Oct 2026 12:12:30 +0700
Subject: [PATCH 1/3] test(gooddata-eval): check the type of each dashboard
 chart

An expected chart may state `chart_type`. The dashboard evaluator reads
the type from the chart body in the response's references, because a
widget names only the chart id, and publishes
`dashboard_chart_types_correct` when a case states one. A title alone
cannot catch a chart authored as another type than the one asked for.
An unknown type is rejected before the run.

JIRA: QA-29722
risk: nonprod

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../core/agentic/dashboard_skill.py           |  77 +++++++++++-
 .../src/gooddata_eval/core/scoring.py         |  35 ++++++
 .../tests/test_agentic_dashboard_skill.py     | 112 ++++++++++++++++++
 3 files changed, 223 insertions(+), 1 deletion(-)

diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py
index 9bbe130db..8819b1abf 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py
@@ -40,6 +40,7 @@
     build_latency_breakdown,
     shift_and_index_events,
 )
+from gooddata_eval.core.scoring import AAC_VISUALIZATION_TYPES, same_viz_type
 from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
 
 _DEFAULT_K = 1
@@ -582,6 +583,64 @@ def _check_date_bindings(widgets: list[dict], new_ids: set[str], expected: list[
     return failures
 
 
+def _has_chart_types(expected_output: dict) -> bool:
+    """Whether any expected chart states the type it must be placed as."""
+    return any(isinstance(v, dict) and "chart_type" in v for v in expected_output.get("visualizations") or [])
+
+
+def _bodies_by_id(*references: dict) -> dict[str, dict]:
+    """Every chart body the response's references carry, existing and authored, by chart id."""
+    bodies: dict[str, dict] = {}
+    for refs in references:
+        for key in ("visualizations", "new_visualizations"):
+            for body in refs.get(key) or []:
+                if isinstance(body, dict) and body.get("id"):
+                    bodies[body["id"]] = body
+    return bodies
+
+
+def _check_chart_types(
+    widgets: list[dict], new_ids: set[str], expected: list[dict], bodies: dict[str, dict]
+) -> tuple[list[str], list[str]]:
+    """Every widget of a chart that states a ``chart_type`` must show a chart of exactly that type.
+
+    A widget names only the chart id, so the type is read from the chart body the references
+    carry. The title alone cannot catch a chart authored as another type than the one asked
+    for, which is the silent swap GDAI-2054 rules out.
+
+    Returns the failures and the notes. A widget whose chart has no body in the references
+    cannot be checked: gen-ai carries a body only when it could resolve one, so a missing
+    body is a platform fault, noted as ``_check_references`` notes a missing id, and not
+    scored as the agent's.
+    """
+    failures: list[str] = []
+    notes: list[str] = []
+    for exp in expected:
+        if not isinstance(exp, dict) or "chart_type" not in exp:
+            continue
+        wanted = exp["chart_type"]
+        label = exp.get("title") if exp.get("id") is None else exp.get("id")
+        candidates = _matching_widgets(widgets, new_ids, exp)
+        if not candidates:
+            failures.append(f"cannot check the type of chart {label!r}: it is not on the dashboard")
+            continue
+        types: list[str] = []
+        for widget in candidates:
+            chart_type = (bodies.get(widget.get("visualization")) or {}).get("type")
+            if isinstance(chart_type, str):
+                types.append(chart_type)
+            else:
+                notes.append(
+                    f"cannot check the type of chart {label!r}: the response's references carry no body "
+                    f"for {widget.get('visualization')!r}"
+                )
+        # Every placement, as for date bindings: one right placement must not hide a wrong one.
+        if not all(same_viz_type(wanted, t) for t in types):
+            found = ", ".join(repr(t) for t in types)
+            failures.append(f"chart {label!r} is {found}, expected {wanted!r}")
+    return failures, notes
+
+
 def _tabs_of(dashboard: dict) -> list[dict]:
     """The dashboard's tabs, reading a document with root sections as its single tab."""
     tabs = dashboard.get("tabs")
@@ -936,6 +995,8 @@ def _validate_layout_expectations(expected_output: dict) -> None:
     for chart in expected_output.get("visualizations") or []:
         if isinstance(chart, dict) and "date_dataset" in chart and not _is_dataset_or_null(chart["date_dataset"]):
             raise ValueError(f"date_dataset must name a date dataset or be null, got {chart!r}")
+        if isinstance(chart, dict) and "chart_type" in chart and chart["chart_type"] not in AAC_VISUALIZATION_TYPES:
+            raise ValueError(f"chart_type must be an AAC visualization type, got {chart!r}")
 
     if "tabs" in expected_output:
         tabs = expected_output["tabs"]
@@ -1037,6 +1098,7 @@ class _Applies:
     tab_filters: bool = False
     date_filters: bool = False
     preserved_widgets: bool = False
+    chart_types: bool = False
 
     @classmethod
     def of(cls, expected_output: dict) -> _Applies:
@@ -1049,6 +1111,7 @@ def of(cls, expected_output: dict) -> _Applies:
             tab_filters="tab_filters" in expected_output,
             date_filters="date_filters" in expected_output,
             preserved_widgets="preserved_widgets" in expected_output,
+            chart_types=_has_chart_types(expected_output),
         )
 
 
@@ -1089,6 +1152,7 @@ class DashboardEvaluation:
     tab_filters_correct: bool = False
     date_filters_correct: bool = False
     widgets_preserved: bool = False
+    chart_types_correct: bool = False
     failures: list[str] = field(default_factory=list)
     notes: list[str] = field(default_factory=list)
 
@@ -1129,6 +1193,8 @@ def strict_checks(self) -> dict[str, bool]:
             checks["dashboard_date_filters_correct"] = self.date_filters_correct
         if self.applies.preserved_widgets:
             checks["dashboard_widgets_preserved"] = self.widgets_preserved
+        if self.applies.chart_types:
+            checks["dashboard_chart_types_correct"] = self.chart_types_correct
         return checks
 
     @property
@@ -1280,12 +1346,14 @@ def evaluate_dashboard_response(
             | {v.get("id") for v in patch_references.get("visualizations") or [] if v.get("id")}
             | new_ids
         )
+        bodies = _bodies_by_id(base_references, patch_references)
     else:
         if actual_saved_id is not None:
             saved_failures.append(f"a new draft must not be saved yet, but it reports id {actual_saved_id!r}")
         document = base
         new_ids = {v.get("id") for v in base_references.get("new_visualizations") or [] if v.get("id")}
         known_ids = {v.get("id") for v in base_references.get("visualizations") or [] if v.get("id")} | new_ids
+        bodies = _bodies_by_id(base_references)
 
     if document is None:
         return DashboardEvaluation(
@@ -1337,6 +1405,11 @@ def evaluate_dashboard_response(
     preserved_failures = (
         _check_preserved_widgets(widgets, expected_output["preserved_widgets"]) if applies.preserved_widgets else []
     )
+    chart_type_failures, chart_type_notes = (
+        _check_chart_types(widgets, new_ids, expected_output.get("visualizations") or [], bodies)
+        if applies.chart_types
+        else ([], [])
+    )
 
     return DashboardEvaluation(
         drafted=True,
@@ -1356,6 +1429,7 @@ def evaluate_dashboard_response(
         tab_filters_correct=not tab_filter_failures,
         date_filters_correct=not date_filter_failures,
         widgets_preserved=not preserved_failures,
+        chart_types_correct=not chart_type_failures,
         failures=[
             *chart_failures,
             *saved_failures,
@@ -1366,9 +1440,10 @@ def evaluate_dashboard_response(
             *tab_filter_failures,
             *date_filter_failures,
             *preserved_failures,
+            *chart_type_failures,
             *new_failures,
         ],
-        notes=[*title_notes, *reference_notes],
+        notes=[*title_notes, *reference_notes, *chart_type_notes],
     )
 
 
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py
index 8c4e4b9b6..eb9c2048e 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py
@@ -21,6 +21,36 @@
 }
 
 
+# Every `type` the AAC visualization schema defines. An expectation naming anything else is a typo.
+AAC_VISUALIZATION_TYPES = frozenset(
+    {
+        "table",
+        "headline_chart",
+        "bar_chart",
+        "column_chart",
+        "line_chart",
+        "area_chart",
+        "pie_chart",
+        "donut_chart",
+        "scatter_chart",
+        "bubble_chart",
+        "treemap_chart",
+        "pyramid_chart",
+        "funnel_chart",
+        "heatmap_chart",
+        "bullet_chart",
+        "waterfall_chart",
+        "dependency_wheel_chart",
+        "sankey_chart",
+        "geo_chart",
+        "geo_area_chart",
+        "repeater_chart",
+        "radar_chart",
+        "combo_chart",
+    }
+)
+
+
 @dataclass
 class FilterScores:
     date_ok: bool
@@ -387,6 +417,11 @@ def _normalize_viz_type(raw_type: str) -> str:
     return _AAC_TYPE_MAP.get(raw_type, raw_type.replace("_chart", "").upper())
 
 
+def same_viz_type(expected: str, actual: str) -> bool:
+    """Whether two chart-type names denote the same type, spelled either way."""
+    return _normalize_viz_type(expected) == _normalize_viz_type(actual)
+
+
 def check_viz_type(expected: CreatedVisualization, actual: CreatedVisualization) -> bool:
     if not expected.type:
         return True
diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py
index 80aa96692..1688e33c2 100644
--- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py
+++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py
@@ -5,6 +5,7 @@
 
 import pytest
 from gooddata_eval.core.agentic.dashboard_skill import (
+    DashboardEvaluation,
     DashboardSkillAssertionError,
     _date_range_text,
     _extract_tool_result,
@@ -1236,6 +1237,117 @@ def test_a_malformed_date_dataset_is_rejected_up_front(self):
         client.send_message.assert_not_called()
 
 
+class TestChartTypes:
+    """GDAI-2054: a chart asked for by type is placed as that type, never as a silent swap.
+    The widget names only the chart id, so the type comes from the body in the references."""
+
+    _WATERFALL = "authored-waterfall"
+
+    def _create(self, authored_type: str, expected_type: str = "waterfall_chart") -> DashboardEvaluation:
+        part = _dashboard_part(
+            [_widget("Gross Profit Bridge", self._WATERFALL)],
+            date_filter=_THIS_YEAR_FILTER,
+            visualizations=[],
+            new_visualizations=[self._WATERFALL],
+        )
+        part["references"]["new_visualizations"][0]["type"] = authored_type
+        expected = {
+            **_DC05_EXPECTED,
+            "visualizations": [{"id": None, "title": "Gross Profit Bridge", "chart_type": expected_type}],
+            "min_new_visualizations": 1,
+        }
+        expected.pop("date_range")
+        return evaluate_dashboard_response(_draft_result(new_visualization_count=1), part, expected, True)
+
+    def test_an_authored_chart_of_the_requested_type_passes(self) -> None:
+        ev = self._create("waterfall_chart")
+        assert ev.strict_checks["dashboard_chart_types_correct"] is True
+        assert ev.strict_pass, ev.failures
+
+    def test_an_authored_chart_of_another_type_fails(self) -> None:
+        ev = self._create("column_chart")
+        assert ev.strict_checks["dashboard_chart_types_correct"] is False
+        assert any("is 'column_chart', expected 'waterfall_chart'" in f for f in ev.failures)
+
+    def test_an_existing_chart_is_typed_from_the_visualizations_reference(self) -> None:
+        part = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER)
+        expected = {
+            **_DC05_EXPECTED,
+            "visualizations": [{"id": _TOTAL_CUSTOMERS, "title": "Total Customers", "chart_type": "headline_chart"}],
+        }
+        ev = evaluate_dashboard_response(_draft_result(), part, expected, True)
+        assert ev.chart_types_correct, ev.failures
+
+    def test_an_edit_reads_the_type_from_the_patch_references(self) -> None:
+        new_widget = _widget("Net Sales Flow", "authored-sankey")
+        patch = _patch_part(
+            [{"op": "add", "path": "/sections/0/widgets/-", "value": new_widget}],
+            new_visualizations=["authored-sankey"],
+        )
+        patch["patch"]["references"]["new_visualizations"][0]["type"] = "sankey_chart"
+        expected = {
+            "type": "dashboardPatch",
+            "saved_dashboard_id": _OVERVIEW_DASHBOARD,
+            "visualizations": [{"id": None, "title": "Net Sales Flow", "chart_type": "sankey_chart"}],
+            "min_new_visualizations": 1,
+        }
+        ev = evaluate_dashboard_response(_patch_result(1), _edit_base_part(), expected, True, patch_part=patch)
+        assert ev.strict_checks["dashboard_chart_types_correct"] is True, ev.failures
+
+    def test_a_chart_not_on_the_dashboard_cannot_pass_the_type_check(self) -> None:
+        part = _dashboard_part([_widget("Active Customers", _ACTIVE_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER)
+        expected = {
+            **_DC05_EXPECTED,
+            "visualizations": [{"id": None, "title": "Gross Profit Bridge", "chart_type": "waterfall_chart"}],
+        }
+        ev = evaluate_dashboard_response(_draft_result(), part, expected, True)
+        assert not ev.chart_types_correct
+        assert any("cannot check the type of chart 'Gross Profit Bridge'" in f for f in ev.failures)
+
+    @pytest.mark.parametrize("reference", [None, {"id": _TOTAL_CUSTOMERS}], ids=["absent", "untyped"])
+    def test_a_chart_without_a_body_is_noted_not_scored(self, reference: dict | None) -> None:
+        """gen-ai carries a body only when it could resolve one: a missing body is a platform fault."""
+        part = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER)
+        part["references"]["visualizations"] = [reference] if reference else []
+        expected = {
+            **_DC05_EXPECTED,
+            "visualizations": [{"id": _TOTAL_CUSTOMERS, "title": "Total Customers", "chart_type": "bar_chart"}],
+        }
+        ev = evaluate_dashboard_response(_draft_result(), part, expected, True)
+        assert ev.strict_checks["dashboard_chart_types_correct"] is True, ev.failures
+        assert any(f"carry no body for {_TOTAL_CUSTOMERS!r}" in note for note in ev.notes)
+
+    def test_only_the_charts_that_state_a_type_are_checked(self) -> None:
+        part = _dashboard_part(
+            [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)],
+            date_filter=_THIS_YEAR_FILTER,
+        )
+        expected = {
+            **_DC05_EXPECTED,
+            "visualizations": [
+                {"id": _TOTAL_CUSTOMERS, "title": "Total Customers", "chart_type": "headline_chart"},
+                {"id": _ACTIVE_CUSTOMERS, "title": "Active Customers"},
+            ],
+        }
+        ev = evaluate_dashboard_response(_draft_result(), part, expected, True)
+        assert ev.strict_checks["dashboard_chart_types_correct"] is True, ev.failures
+
+    def test_a_case_without_chart_types_does_not_publish_the_check(self) -> None:
+        part = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER)
+        ev = evaluate_dashboard_response(_draft_result(), part, _DC05_EXPECTED, skill_activated=True)
+        assert "dashboard_chart_types_correct" not in ev.strict_checks
+
+    def test_an_unknown_chart_type_is_rejected_up_front(self) -> None:
+        client = MagicMock()
+        expected = {
+            **_DC05_EXPECTED,
+            "visualizations": [{"id": _TOTAL_CUSTOMERS, "title": "T", "chart_type": "waterfall"}],
+        }
+        with pytest.raises(ValueError, match="chart_type must be an AAC visualization type"):
+            _run_with(client, expected)
+        client.send_message.assert_not_called()
+
+
 class TestTabs:
     """GDAI-2426: the tabs the user named, in their order; a plain request stays on one tab."""
 

From 921eff137c4a7c4581636e83c5654ae4488934d0 Mon Sep 17 00:00:00 2001
From: huyenthanh09 <huyen.nguyen@gooddata.com>
Date: Thu, 22 Oct 2026 12:15:56 +0700
Subject: [PATCH 2/3] test(gooddata-eval): score the buckets of the extended
 chart types

The visualization evaluator now reads every bucket the extended chart
types use:
- `from` and `to` of a sankey or dependency wheel count as dimensions,
  and `size_by` of a bubble chart as a metric;
- a metric a repeater writes into `columns` counts as a metric, so the
  chart scores alike whichever bucket the agent put it in;
- a null metric holds the x position of a bubble with no x measure.

Three opt-in switches on an expected chart sharpen the check:
- `bucket_strict` compares bucket by bucket and publishes
  `assertion-vis-buckets`, so swapped heatmap rows and columns or a
  reversed sankey no longer pass on the union of the buckets;
- `metrics_ordered` compares the metrics by position, for bullet roles
  and bubble axes;
- an `axis` on an expected metric entry must match the combo axis.

JIRA: QA-29722
risk: nonprod

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../core/agentic/visualization.py             |  16 ++
 .../core/dataset/from_insights.py             |   7 +-
 .../core/evaluators/visualization.py          |  32 +++-
 .../src/gooddata_eval/core/models.py          |  24 ++-
 .../src/gooddata_eval/core/scoring.py         | 121 +++++++++++-
 .../tests/test_visualization_evaluator.py     | 177 ++++++++++++++++++
 6 files changed, 360 insertions(+), 17 deletions(-)

diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py
index 0dd553353..b77738902 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py
@@ -393,6 +393,17 @@ def _run_detail(run: RunResult) -> dict:
     }
 
 
+def _buckets_line(ev: EvaluationResult) -> str:
+    """The bucket-by-bucket verdict for the failure summary, or "" when the item does not ask for it."""
+    lines = "".join(f"    {problem}\n" for problem in ev.metric_layout_problems)
+    if ev.buckets_correct is None:
+        return lines
+    lines += f"  Buckets Correct       : {ev.buckets_correct}\n"
+    if not ev.buckets_correct:
+        lines += f"    expected : {ev.expected_buckets}\n    actual   : {ev.actual_buckets}\n"
+    return lines
+
+
 def _filter_diff(category: str, ev: EvaluationResult) -> str:
     """Expected-vs-actual lines for one filter category, or "" when they matched.
 
@@ -483,12 +494,16 @@ def _write_scores(ctx: RunTraceContext) -> None:
                 }
                 if ev.requires_execution:
                     strict_checks["assertion-vis-executed"] = ev.executed
+                if ev.buckets_correct is not None:
+                    strict_checks["assertion-vis-buckets"] = ev.buckets_correct
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     ctx.score(tid, name="assertion-cross-ref-valid", value=ev.cross_ref_valid, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-metric", value=ev.metrics_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-dimensions", value=ev.dimensions_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN")
+                    if ev.buckets_correct is not None:
+                        ctx.score(tid, name="assertion-vis-buckets", value=ev.buckets_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN")
                     # Scored on every item, gating or not, so the run rate is visible per model.
                     ctx.score(tid, name="assertion-vis-executed", value=ev.executed, data_type="BOOLEAN")
@@ -587,6 +602,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
             f"    attribute : {ev.filter_attribute_score}\n"
             f"{_filter_diff('attribute', ev)}"
             f"  Viz Type Hard         : {ev.viz_type_hard}\n"
+            f"{_buckets_line(ev)}"
             f"  Executed              : {ev.executed}{' (required)' if ev.requires_execution else ''}\n"
             f"  Stated Value Matches  : {ev.stated_value_matches} (reported only)\n"
             "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/from_insights.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/from_insights.py
index 332dc48c7..dd19314a4 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/from_insights.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/from_insights.py
@@ -146,13 +146,14 @@ def list_ids(directory: Path) -> set[str]:
     return ids
 
 
-def aliases(bucket: list[AacBucketRef | str]) -> list[str]:
+def aliases(bucket: list[AacBucketRef | str] | list[AacBucketRef | str | None]) -> list[str]:
     """A bucket's field aliases.
 
     A bucket item is an alias, or an `AacBucketRef` carrying `format`, `axis`, `totals`
-    or `display_as` -- presentation, which nothing here or in the scorer reads.
+    or `display_as` -- presentation, which nothing here reads. A null metric only holds a
+    position and names no field.
     """
-    return [item if isinstance(item, str) else item.field for item in bucket]
+    return [item if isinstance(item, str) else item.field for item in bucket if item is not None]
 
 
 def _reject_unscorable(spec: CreatedVisualization) -> None:
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py
index 19b7bb0b8..d184e6d6d 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py
@@ -2,7 +2,7 @@
 """Agentic visualization evaluator — ported from gdc-nas tavern-e2e app/vis_agentic.py."""
 
 import re
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, field, replace
 from datetime import date
 from typing import Any
 
@@ -15,7 +15,10 @@
     timeline_detail,
 )
 from gooddata_eval.core.scoring import (
+    bucket_layout,
+    check_buckets,
     check_filters,
+    check_metric_layout,
     check_sorts,
     check_viz_type,
     get_dimension_uri_set,
@@ -61,6 +64,13 @@ class EvaluationResult:
     # Whether a number in the reply matches a value the chart returned. None when the reply
     # states no number. Reported only: reading numbers out of prose has false hits.
     stated_value_matches: bool | None = None
+    # Whether each field sits in the expected bucket. None when the expectation does not ask
+    # (`bucket_strict`), so the check neither gates nor shows.
+    buckets_correct: bool | None = None
+    expected_buckets: dict[str, list[str]] | None = None
+    actual_buckets: dict[str, list[str]] | None = None
+    # Why `metrics_correct` is False although the metric set matched: order or axis.
+    metric_layout_problems: list[str] = field(default_factory=list)
 
     @property
     def strict_pass(self) -> bool:
@@ -73,6 +83,7 @@ def strict_pass(self) -> bool:
             and self.sorts_correct
             and self.viz_type_hard
             and (self.executed or not self.requires_execution)
+            and self.buckets_correct is not False
         )
 
     @property
@@ -85,6 +96,7 @@ def strict_checks_passed_count(self) -> int:
                 self.filters_correct,
                 self.sorts_correct,
                 self.viz_type_hard,
+                self.buckets_correct is True,
             ]
         )
 
@@ -257,15 +269,18 @@ def _evaluate_visualization(
             actual_filters={category: values.copy() for category, values in _NO_FILTERS.items()},
             expected_sorts=normalized_sorts(expected),
             actual_sorts=[],
+            buckets_correct=False if expected.bucket_strict else None,
+            expected_buckets=bucket_layout(expected) if expected.bucket_strict else None,
         )
     cross_ref_valid, cross_ref_errors = validate_cross_references(actual)
     act_metric_uris = get_metric_uri_set(actual)
     act_dim_uris = get_dimension_uri_set(actual)
     filter_scores = check_filters(expected, actual, today)
+    metric_layout_problems = check_metric_layout(expected, actual)
     return EvaluationResult(
         visualization_created=True,
         cross_ref_valid=cross_ref_valid,
-        metrics_correct=act_metric_uris == exp_metric_uris,
+        metrics_correct=act_metric_uris == exp_metric_uris and not metric_layout_problems,
         dimensions_correct=act_dim_uris == exp_dim_uris,
         filters_correct=filter_scores.all_ok,
         sorts_correct=check_sorts(expected, actual),
@@ -283,6 +298,10 @@ def _evaluate_visualization(
         actual_sorts=normalized_sorts(actual),
         expected_filters=normalized_filters(expected, today),
         actual_filters=normalized_filters(actual, today),
+        buckets_correct=check_buckets(expected, actual),
+        expected_buckets=bucket_layout(expected) if expected.bucket_strict else None,
+        actual_buckets=bucket_layout(actual) if expected.bucket_strict else None,
+        metric_layout_problems=metric_layout_problems,
     )
 
 
@@ -349,6 +368,15 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict:
         "actual_filters": ev.actual_filters,
         "expected_sorts": ev.expected_sorts,
         "actual_sorts": ev.actual_sorts,
+        **({"metric_layout_problems": ev.metric_layout_problems} if ev.metric_layout_problems else {}),
+        # Top level only when the expectation asks for it: a check the item does not gate on
+        # must not lift `quality_score`, which counts every top-level boolean.
+        **({"buckets_correct": ev.buckets_correct} if ev.buckets_correct is not None else {}),
+        **(
+            {"buckets": {"expected": ev.expected_buckets, "actual": ev.actual_buckets}}
+            if ev.expected_buckets is not None
+            else {}
+        ),
         # Nested: `quality_score` counts every top-level boolean, and a check an item does not
         # gate on must not lower it. `executed` joins the top level only when it gates.
         "execution": {
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py
index 5bcdd0045..dd7da849d 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/models.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py
@@ -9,7 +9,7 @@
 from enum import Enum
 from typing import Any
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, field_validator, model_serializer
 
 from gooddata_eval.core.timing import PhaseTimings
 
@@ -96,7 +96,7 @@ def _coerce_sort_by(cls, v: object) -> object:
 class CreatedVisualization(BaseModel):
     """Visualization in the AAC format (agent output and dataset expected output)."""
 
-    model_config = ConfigDict(extra="ignore")
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
 
     # Optional on purpose: nothing scores on it, and the agent sometimes omits it. A
     # required field here turns a scorable chart into a parse error and an errored item.
@@ -104,18 +104,34 @@ class CreatedVisualization(BaseModel):
     title: str | None = None
     type: str
     query: AacQuery
-    metrics: list[AacBucketRef | str] = Field(default_factory=list)
+    # A null entry holds a position: a bubble chart with no x measure is `[null, y]`.
+    metrics: list[AacBucketRef | str | None] = Field(default_factory=list)
     view_by: list[AacBucketRef | str] = Field(default_factory=list)
     segment_by: list[AacBucketRef | str] = Field(default_factory=list)
     rows: list[AacBucketRef | str] = Field(default_factory=list)
     columns: list[AacBucketRef | str] = Field(default_factory=list)
+    # The metric that sizes each bubble of a bubble chart.
+    size_by: list[AacBucketRef | str] = Field(default_factory=list)
+    # The source and the destination of a sankey or dependency wheel: one bucket item each, not a list.
+    from_: AacBucketRef | str | None = Field(default=None, alias="from")
+    to: AacBucketRef | str | None = None
     config: dict | None = None
+    # Expectation switches. An agent never sends them, and a recorded output leaves them out.
+    bucket_strict: bool = Field(default=False, exclude=True)
+    metrics_ordered: bool = Field(default=False, exclude=True)
 
-    @field_validator("metrics", "view_by", "segment_by", "rows", "columns", mode="before")
+    @field_validator("metrics", "view_by", "segment_by", "rows", "columns", "size_by", mode="before")
     @classmethod
     def _coerce_list_fields(cls, v: object) -> object:
         return v if v is not None else []
 
+    @model_serializer(mode="wrap")
+    def _dump_from_under_its_aac_name(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]:
+        # `from` is a keyword, so the field is `from_`. A dump without `by_alias` must still
+        # write the AAC key: a dataset or a fixture holding `from_` loses the source bucket in
+        # every reader that does not know the Python name.
+        return {"from" if key == "from_" else key: value for key, value in handler(self).items()}
+
 
 class CreatedVisualizations(BaseModel):
     model_config = ConfigDict(extra="ignore")
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py
index eb9c2048e..f8f6fd503 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py
@@ -80,23 +80,128 @@ def resolve_alias_to_uri(alias: str, fields: Mapping[str, AacQueryField | str |
     return field.using
 
 
-def _resolve_bucket_to_uri_set(bucket: list[AacBucketRef | str], fields: dict[str, AacQueryField | str]) -> set[str]:
-    uris: set[str] = set()
-    for ref in bucket:
-        alias = ref.field if isinstance(ref, AacBucketRef) else ref
-        uris.add(canonical_date_uri(resolve_alias_to_uri(alias, fields)))
-    return uris
+BucketItem = AacBucketRef | str
+
+
+def _ref_uri(ref: BucketItem, fields: Mapping[str, AacQueryField | str]) -> str:
+    alias = ref.field if isinstance(ref, AacBucketRef) else ref
+    return canonical_date_uri(resolve_alias_to_uri(alias, fields))
+
+
+def _resolve_bucket_to_uri_set(
+    bucket: list[BucketItem] | list[BucketItem | None], fields: dict[str, AacQueryField | str]
+) -> set[str]:
+    return {_ref_uri(ref, fields) for ref in bucket if ref is not None}
+
+
+def _is_measure(ref: BucketItem, fields: dict[str, AacQueryField | str]) -> bool:
+    """Whether a bucket item names a metric, a fact, or a field the chart aggregates itself."""
+    alias = ref.field if isinstance(ref, AacBucketRef) else ref
+    field = fields.get(alias)
+    if isinstance(field, AacQueryField) and field.aggregation:
+        return True
+    return resolve_alias_to_uri(alias, fields).startswith(("metric/", "fact/"))
+
+
+def _column_split(viz: CreatedVisualization) -> tuple[list[BucketItem], list[BucketItem]]:
+    """`columns` split into its measures and its attributes.
+
+    A table's `columns` holds attributes only. A repeater writes its metrics there too, as
+    `{field, display_as}` entries next to a clone of its row attribute, and the same chart
+    comes back with the metrics under `metrics` on another run. Splitting by what each item
+    names scores both spellings alike.
+    """
+    measures = [ref for ref in viz.columns if _is_measure(ref, viz.query.fields)]
+    attributes = [ref for ref in viz.columns if not _is_measure(ref, viz.query.fields)]
+    return measures, attributes
 
 
 def get_metric_uri_set(viz: CreatedVisualization) -> set[str]:
-    return _resolve_bucket_to_uri_set(viz.metrics, viz.query.fields)
+    measures_in_columns, _ = _column_split(viz)
+    return _resolve_bucket_to_uri_set(viz.metrics + viz.size_by + measures_in_columns, viz.query.fields)
 
 
 def get_dimension_uri_set(viz: CreatedVisualization) -> set[str]:
-    all_dim_buckets = viz.view_by + viz.segment_by + viz.rows + viz.columns
+    _, attributes_in_columns = _column_split(viz)
+    ends = [ref for ref in (viz.from_, viz.to) if ref is not None]
+    all_dim_buckets = viz.view_by + viz.segment_by + viz.rows + attributes_in_columns + ends
     return _resolve_bucket_to_uri_set(all_dim_buckets, viz.query.fields)
 
 
+def bucket_layout(viz: CreatedVisualization) -> dict[str, list[str]]:
+    """Each bucket's fields, as uris, for comparing bucket by bucket.
+
+    Sorted lists rather than sets so the layout reads in a report. An attribute in `columns`
+    that already sits in `rows` or `view_by` is left out: the repeater adds that clone of
+    its row attribute on its own, so it says nothing about where the agent put the field.
+    """
+    fields = viz.query.fields
+    _, attributes_in_columns = _column_split(viz)
+    placed = _resolve_bucket_to_uri_set(viz.rows + viz.view_by, fields)
+    columns = _resolve_bucket_to_uri_set(attributes_in_columns, fields) - placed
+    return {
+        "view_by": sorted(_resolve_bucket_to_uri_set(viz.view_by, fields)),
+        "segment_by": sorted(_resolve_bucket_to_uri_set(viz.segment_by, fields)),
+        "rows": sorted(_resolve_bucket_to_uri_set(viz.rows, fields)),
+        "columns": sorted(columns),
+        "size_by": sorted(_resolve_bucket_to_uri_set(viz.size_by, fields)),
+        "from": [_ref_uri(viz.from_, fields)] if viz.from_ is not None else [],
+        "to": [_ref_uri(viz.to, fields)] if viz.to is not None else [],
+    }
+
+
+def check_buckets(expected: CreatedVisualization, actual: CreatedVisualization) -> bool | None:
+    """Whether every field sits in the bucket the expectation puts it in; None when not asked.
+
+    The dimension check compares the union of the buckets, so a heatmap with its rows and
+    columns swapped, a treemap segment moved into `view_by`, or a sankey with source and
+    destination reversed would all pass it.
+    """
+    if not expected.bucket_strict:
+        return None
+    return bucket_layout(actual) == bucket_layout(expected)
+
+
+def ordered_metric_uris(viz: CreatedVisualization) -> list[str | None]:
+    """The chart's `metrics` in order, a null entry kept as None."""
+    return [None if ref is None else _ref_uri(ref, viz.query.fields) for ref in viz.metrics]
+
+
+_PRIMARY_AXIS = "primary"
+
+
+def metric_axes(viz: CreatedVisualization) -> dict[str, str]:
+    """The axis each metric of a combo chart is drawn on, by metric uri."""
+    axes: dict[str, str] = {}
+    for ref in viz.metrics:
+        if isinstance(ref, AacBucketRef):
+            axis = (ref.model_extra or {}).get("axis")
+            if isinstance(axis, str):
+                axes[_ref_uri(ref, viz.query.fields)] = axis
+    return axes
+
+
+def check_metric_layout(expected: CreatedVisualization, actual: CreatedVisualization) -> list[str]:
+    """What is wrong with the order and the axes of the metrics, when the expectation states them.
+
+    Order is checked under `metrics_ordered`: a bullet chart reads actual, target and
+    comparative by position, and a bubble chart x and y. An axis is checked for every
+    expected metric entry that names one. An actual metric that names no axis, written as a
+    plain alias or as an entry without `axis`, is drawn on the primary axis.
+    """
+    problems: list[str] = []
+    if expected.metrics_ordered and ordered_metric_uris(actual) != ordered_metric_uris(expected):
+        problems.append(f"metric order {ordered_metric_uris(actual)}, expected {ordered_metric_uris(expected)}")
+    expected_axes = metric_axes(expected)
+    if expected_axes:
+        present = set(ordered_metric_uris(actual))
+        actual_axes = {uri: _PRIMARY_AXIS for uri in present if uri is not None} | metric_axes(actual)
+        wrong = {uri: actual_axes.get(uri) for uri, axis in expected_axes.items() if actual_axes.get(uri) != axis}
+        if wrong:
+            problems.append(f"metric axes {wrong}, expected {expected_axes}")
+    return problems
+
+
 def uri_to_display_name(uri: str) -> str:
     """Convert 'metric/net_sales' -> 'net sales', 'label/date.month' -> 'date - month'."""
     last = uri.split("/", 1)[-1]
diff --git a/packages/gooddata-eval/tests/test_visualization_evaluator.py b/packages/gooddata-eval/tests/test_visualization_evaluator.py
index 3d5cac1e6..ae37b4722 100644
--- a/packages/gooddata-eval/tests/test_visualization_evaluator.py
+++ b/packages/gooddata-eval/tests/test_visualization_evaluator.py
@@ -6,6 +6,7 @@
 from gooddata_eval.core.evaluators import get_evaluator
 from gooddata_eval.core.evaluators.visualization import _evaluate_visualization, execution_signals
 from gooddata_eval.core.models import ChatResult, CreatedVisualization, DatasetItem, ToolCallEvent
+from gooddata_eval.core.scoring import get_dimension_uri_set, get_metric_uri_set
 
 
 def _item(expected_viz) -> DatasetItem:
@@ -367,3 +368,179 @@ def test_a_ref_that_is_not_a_string_is_skipped_not_raised():
         result='{"success":true,"data":{"rows":[]}}',
     )
     assert execution_signals([_CREATE, execute], None) == (False, None)
+
+
+# -- Extended chart types (QA-29722): shapes trimmed from ecommerce_demo runs -----------------
+
+_EXT_FIELDS = {
+    "m_net_sales": {"using": "metric/net_sales"},
+    "m_margin": {"using": "metric/gross_profit_margin"},
+    "m_orders": {"using": "metric/of_orders"},
+    "d_category": {"using": "label/product_category"},
+    "d_country": {"using": "label/customer_country"},
+    "d_brand": {"using": "label/product_brand"},
+    "d_month": {"using": "label/date.month"},
+}
+
+
+def _ext(type_: str, **buckets) -> dict:
+    return {"type": type_, "query": {"fields": _EXT_FIELDS, "filter_by": {}}, **buckets}
+
+
+def _score(expected: dict, actual: dict):
+    return _evaluate_visualization(
+        CreatedVisualization.model_validate(expected), CreatedVisualization.model_validate(actual)
+    )
+
+
+def test_a_repeater_scores_the_same_with_its_metric_in_metrics_or_in_columns():
+    expected = _ext("repeater_chart", metrics=["m_net_sales"], rows=["d_category"], view_by=["d_month"])
+    in_metrics = _ext(
+        "repeater_chart",
+        metrics=[{"field": "m_net_sales", "display_as": "line"}],
+        rows=["d_category"],
+        view_by=["d_month"],
+    )
+    in_columns = _ext(
+        "repeater_chart",
+        rows=["d_category"],
+        view_by=["d_month"],
+        columns=["d_category_column", {"field": "m_net_sales", "display_as": "line"}],
+    )
+    in_columns["query"] = {"fields": {**_EXT_FIELDS, "d_category_column": {"using": "label/product_category"}}}
+    for actual in (in_metrics, in_columns):
+        ev = _score({**expected, "bucket_strict": True}, actual)
+        assert ev.metrics_correct, ev.actual_metric_uris
+        assert ev.dimensions_correct, ev.actual_dim_uris
+        assert ev.buckets_correct is True, ev.actual_buckets
+        assert ev.strict_pass
+
+
+def test_sankey_ends_are_dimensions_and_their_direction_is_checked_under_bucket_strict():
+    expected = _ext("sankey_chart", metrics=["m_net_sales"], **{"from": "d_category", "to": "d_country"})
+    right = _score({**expected, "bucket_strict": True}, expected)
+    assert right.expected_dim_uris == {"label/product_category", "label/customer_country"}
+    assert right.strict_pass
+
+    reversed_ends = _ext("sankey_chart", metrics=["m_net_sales"], **{"from": "d_country", "to": "d_category"})
+    loose = _score(expected, reversed_ends)
+    strict = _score({**expected, "bucket_strict": True}, reversed_ends)
+    assert loose.strict_pass, "without bucket_strict the union of the ends is all that counts"
+    assert strict.buckets_correct is False
+    assert not strict.strict_pass
+
+
+def test_a_missing_sankey_destination_fails_the_dimension_check():
+    expected = _ext("sankey_chart", metrics=["m_net_sales"], **{"from": "d_category", "to": "d_country"})
+    no_destination = _ext("sankey_chart", metrics=["m_net_sales"], **{"from": "d_category"})
+    assert not _score(expected, no_destination).dimensions_correct
+
+
+def test_a_heatmap_with_rows_and_columns_swapped_fails_only_under_bucket_strict():
+    expected = _ext("heatmap_chart", metrics=["m_net_sales"], view_by=["d_brand"], segment_by=["d_country"])
+    swapped = _ext("heatmap_chart", metrics=["m_net_sales"], view_by=["d_country"], segment_by=["d_brand"])
+    assert _score(expected, swapped).strict_pass
+    ev = _score({**expected, "bucket_strict": True}, swapped)
+    assert ev.buckets_correct is False
+    assert ev.expected_buckets["view_by"] == ["label/product_brand"]
+    assert ev.actual_buckets["view_by"] == ["label/customer_country"]
+
+
+def test_bubble_size_counts_as_a_metric_and_order_holds_the_x_position():
+    expected = _ext(
+        "bubble_chart",
+        metrics=[None, "m_margin"],
+        size_by=["m_orders"],
+        view_by=["d_brand"],
+        metrics_ordered=True,
+        bucket_strict=True,
+    )
+    right = _ext("bubble_chart", metrics=[None, "m_margin"], size_by=["m_orders"], view_by=["d_brand"])
+    ev = _score(expected, right)
+    assert ev.expected_metric_uris == {"metric/gross_profit_margin", "metric/of_orders"}
+    assert ev.strict_pass, ev.metric_layout_problems
+
+    margin_on_x = _ext("bubble_chart", metrics=["m_margin"], size_by=["m_orders"], view_by=["d_brand"])
+    ev = _score(expected, margin_on_x)
+    assert not ev.metrics_correct
+    assert ev.metric_layout_problems and "metric order" in ev.metric_layout_problems[0]
+
+
+def test_bullet_roles_are_positional_only_when_metrics_ordered():
+    roles = ["m_net_sales", "m_orders", "m_margin"]
+    expected = _ext("bullet_chart", metrics=roles, view_by=["d_category"])
+    shuffled = _ext("bullet_chart", metrics=["m_orders", "m_net_sales", "m_margin"], view_by=["d_category"])
+    assert _score(expected, shuffled).metrics_correct
+    assert not _score({**expected, "metrics_ordered": True}, shuffled).metrics_correct
+
+
+def test_combo_axes_are_checked_for_every_metric_that_names_one():
+    expected = _ext(
+        "combo_chart",
+        metrics=[{"field": "m_net_sales", "axis": "primary"}, {"field": "m_margin", "axis": "secondary"}],
+        view_by=["d_category"],
+    )
+    assert _score(expected, expected).strict_pass
+    both_primary = _ext(
+        "combo_chart",
+        metrics=[{"field": "m_net_sales", "axis": "primary"}, {"field": "m_margin", "axis": "primary"}],
+        view_by=["d_category"],
+    )
+    ev = _score(expected, both_primary)
+    assert not ev.metrics_correct
+    assert "metric axes" in ev.metric_layout_problems[0]
+
+
+def test_combo_axes_read_a_metric_without_an_axis_as_primary():
+    expected = _ext(
+        "combo_chart",
+        metrics=[{"field": "m_net_sales", "axis": "primary"}, {"field": "m_margin", "axis": "secondary"}],
+        view_by=["d_category"],
+    )
+    for primary in ("m_net_sales", {"field": "m_net_sales"}):
+        actual = _ext(
+            "combo_chart", metrics=[primary, {"field": "m_margin", "axis": "secondary"}], view_by=["d_category"]
+        )
+        ev = _score(expected, actual)
+        assert ev.metrics_correct, ev.metric_layout_problems
+    secondary_unstated = _ext("combo_chart", metrics=["m_net_sales", "m_margin"], view_by=["d_category"])
+    ev = _score(expected, secondary_unstated)
+    assert not ev.metrics_correct
+    assert "'metric/gross_profit_margin': 'primary'" in ev.metric_layout_problems[0]
+
+
+def test_a_field_the_chart_aggregates_itself_counts_as_a_metric_in_columns():
+    fields = {**_EXT_FIELDS, "c_orders": {"using": "attribute/order_id", "aggregation": "COUNT"}}
+    repeater = _ext("repeater_chart", rows=["d_category"], columns=[{"field": "c_orders", "display_as": "line"}])
+    repeater["query"] = {"fields": fields, "filter_by": {}}
+    viz = CreatedVisualization.model_validate(repeater)
+    assert get_metric_uri_set(viz) == {"attribute/order_id"}
+    assert get_dimension_uri_set(viz) == {"label/product_category"}
+
+
+def test_a_sankey_source_is_written_under_its_aac_name():
+    """`from` is a Python keyword: the field is `from_`, but no dump may write that name."""
+    viz = CreatedVisualization.model_validate(
+        _ext("sankey_chart", metrics=["m_net_sales"], **{"from": "d_category", "to": "d_country"})
+    )
+    for dumped in (viz.model_dump(exclude_none=True), viz.model_dump(mode="json")):
+        assert dumped["from"] == "d_category"
+        assert "from_" not in dumped
+    assert '"from":"d_category"' in viz.model_dump_json()
+    assert list(viz.model_dump()).index("from") < list(viz.model_dump()).index("to")
+    assert CreatedVisualization.model_validate(viz.model_dump()).from_ == "d_category"
+
+
+def test_bucket_check_is_absent_from_the_detail_unless_asked_for():
+    ev = get_evaluator("visualization")
+    plain = ev.evaluate(_item(_expected()), _chat_result_with(dict(_expected())))
+    assert "buckets_correct" not in plain.detail
+    strict = ev.evaluate(_item({**_expected(), "bucket_strict": True}), _chat_result_with(dict(_expected())))
+    assert strict.detail["buckets_correct"] is True
+
+
+def test_expectation_switches_are_left_out_of_a_recorded_output():
+    viz = CreatedVisualization.model_validate({**_expected(), "bucket_strict": True, "metrics_ordered": True})
+    dumped = viz.model_dump(exclude_none=True)
+    assert "bucket_strict" not in dumped
+    assert "metrics_ordered" not in dumped

From 2800d0ea16705ff6d0e84b117d95322207806136 Mon Sep 17 00:00:00 2001
From: huyenthanh09 <huyen.nguyen@gooddata.com>
Date: Thu, 22 Oct 2026 12:23:32 +0700
Subject: [PATCH 3/3] test(gooddata-eval): check answers without a chart,
 number formats and the chart part

A visualization expectation can now state, beside its candidate charts:
- `expects_visualization: false`: the agent must answer without a chart,
  for example by asking for a part the request leaves out. The run stops
  after one turn and publishes `assertion-vis-no-chart`.
- `chart_optional: true`: either an answer without a chart or a chart
  that matches a candidate is right, for a request the chart type cannot
  draw as asked. The run stops after one turn.
- `answer_must_include` / `answer_must_not_match`: regular expressions
  matched without regard to case against the final answer, published as
  `assertion-vis-answer`. A typographic apostrophe reads as a plain one.
- `format_intents`: how a metric's numbers must read. The agent's own
  `format_example` from the chart tool call is read back against the
  intent and compared as a number, so `#,##0.0M` showing 4,221,301,265.5M
  for "in millions" fails (GDAI-2592). Published as `assertion-vis-format`.
- `require_visualization_part`: the chart must come back as a part of the
  answer, not only as tool arguments. Published as
  `assertion-vis-part-present`.

`expectation_options()` reads and validates these keys before any
request. Every key is opt-in, so existing items score as before. A run
keeps the final answer text, and the failure summary shows it when an
answer check fails.

JIRA: QA-29722
risk: nonprod

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../src/gooddata_eval/cli/agentic_runner.py   |   6 +-
 .../core/agentic/visualization.py             | 136 ++++++++---
 .../core/evaluators/visualization.py          | 216 ++++++++++++++++-
 .../src/gooddata_eval/core/number_format.py   | 177 ++++++++++++++
 .../tests/test_agentic_runner.py              |  52 ++++-
 .../tests/test_agentic_visualization.py       | 219 ++++++++++++++++++
 .../gooddata-eval/tests/test_number_format.py | 174 ++++++++++++++
 .../tests/test_visualization_evaluator.py     | 177 +++++++++++++-
 8 files changed, 1119 insertions(+), 38 deletions(-)
 create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/number_format.py
 create mode 100644 packages/gooddata-eval/tests/test_number_format.py

diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
index 60e9b0d46..e8feb992e 100644
--- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
+++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
@@ -26,7 +26,7 @@
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
-from gooddata_eval.core.evaluators.visualization import requires_execution_of
+from gooddata_eval.core.evaluators.visualization import expectation_options
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 from gooddata_eval.core.timing import TraceRecorder
@@ -119,6 +119,8 @@ def _parse_visualization_expected(expected_output: Any) -> list[CreatedVisualiza
       {"visualization": {...}} or {"visualization": [{...}]}  <- single/multi candidate
       [{"visualization": {...}}, ...]                          <- bare list
     """
+    if isinstance(expected_output, dict) and expected_output.get("expects_visualization") is False:
+        return []
     if isinstance(expected_output, dict):
         raw_list = expected_output.get("expected_outputs")
         if raw_list is not None:
@@ -187,7 +189,7 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
-            requires_execution=requires_execution_of(eo),
+            options=expectation_options(eo),
             k=k,
             gate=gate,
             agent_id=agent_id,
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py
index b77738902..ab0f83de0 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py
@@ -9,7 +9,7 @@
 
 import os
 import time
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 
 from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
@@ -34,10 +34,12 @@
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators.visualization import (
     EvaluationResult,
+    ExpectationOptions,
     _check_visualization_skill_activated,
     _evaluate_against_candidates,
     evaluation_result_detail,
     with_execution,
+    with_options,
 )
 from gooddata_eval.core.langfuse.item_scope import item_scope
 from gooddata_eval.core.models import (
@@ -64,7 +66,8 @@ class RunResult:
     conversation_id: str
     actual_output: CreatedVisualization | None
     eval_result: EvaluationResult
-    best_expected: CreatedVisualization
+    # None for an item that expects no chart.
+    best_expected: CreatedVisualization | None
     total_turns: int
     total_steps: int
     reasoning_steps: list[str] = field(default_factory=list)
@@ -76,6 +79,8 @@ class RunResult:
     exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
     # This run's own wall time, from just before its first chat call to the end of its scoring.
     run_latency_s: float = 0.0
+    # Everything the final turn said, which the answer patterns are matched against.
+    answer_text: str = ""
 
 
 @dataclass
@@ -196,8 +201,13 @@ def _execute_single_run(
     expected_outputs: list[CreatedVisualization],
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     requires_execution: bool = False,
+    options: ExpectationOptions | None = None,
 ) -> RunResult:
     """Drive one full multi-turn conversation and evaluate the result."""
+    options = _effective_options(options, requires_execution)
+    one_turn = not options.expects_visualization or options.chart_optional
+    if one_turn:
+        max_iterations = min(max_iterations, 1)
     total_turns = 0
     total_steps = 0
     all_tool_call_events: list[ToolCallEvent] = []
@@ -207,7 +217,8 @@ def _execute_single_run(
     turn_offset = 0.0  # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time
     reasoning_index_offset = 0  # ditto for ReasoningStepEvent.index, which also restarts per turn
     tool_index_offset = 0  # ditto for ToolCallEvent.index
-    simulated_response_guide = expected_outputs[0]  # primary candidate guides the simulated user
+    # The primary candidate guides the simulated user. An item with no chart never replies.
+    simulated_response_guide = expected_outputs[0] if expected_outputs else None
 
     # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that
     # simply runs out of range() is labelled correctly with no trailing else.
@@ -258,7 +269,11 @@ def _execute_single_run(
         if not response_text:
             exit_reason = LoopExit.AGENT_SILENT
             break
-        if iteration >= max_iterations - 1:
+        if one_turn:
+            # Answering without a chart is an outcome this item accepts.
+            exit_reason = LoopExit.SUCCESS
+            break
+        if iteration >= max_iterations - 1 or simulated_response_guide is None:
             break
 
         try:
@@ -288,7 +303,11 @@ def _execute_single_run(
         actual_output = current_result.created_visualizations.objects[0]
 
     eval_result, best_expected = _evaluate_against_candidates(expected_outputs, actual_output, skill_activated)
-    eval_result = with_execution(eval_result, all_tool_call_events, current_result.text_response, requires_execution)
+    eval_result = with_execution(
+        eval_result, all_tool_call_events, current_result.text_response, options.requires_execution
+    )
+    answer_text = render_answer_text(current_result)
+    eval_result = with_options(eval_result, options, actual_output, all_tool_call_events, answer_text)
 
     return RunResult(
         conversation_id=conversation_id,
@@ -303,9 +322,16 @@ def _execute_single_run(
         reasoning_step_events=all_reasoning_step_events,
         exit_reason=exit_reason,
         run_latency_s=time.monotonic() - run_started,
+        answer_text=answer_text,
     )
 
 
+def _effective_options(options: ExpectationOptions | None, requires_execution: bool) -> ExpectationOptions:
+    """``options`` with the separate ``requires_execution`` argument folded in."""
+    options = options or ExpectationOptions()
+    return replace(options, requires_execution=True) if requires_execution else options
+
+
 def run_agentic_visualization(
     host: str,
     token: str,
@@ -319,6 +345,7 @@ def run_agentic_visualization(
     agent_id: str | None = None,
     user_context: dict | None = None,
     requires_execution: bool = False,
+    options: ExpectationOptions | None = None,
 ) -> AgenticRunSummary:
     """Run K independent conversations and return evaluation results.
 
@@ -328,7 +355,18 @@ def run_agentic_visualization(
     conversations created by this function are deleted on completion.
 
     ``requires_execution`` fails a run whose agent built the chart but never ran it.
+
+    Raises:
+        ValueError: ``expected_outputs`` and ``options.expects_visualization`` disagree. An
+            empty candidate list is only right for an item that expects no chart, and the
+            scorer would otherwise read the default ``expects_visualization`` and fail it.
     """
+    expects_visualization = (options or ExpectationOptions()).expects_visualization
+    if expects_visualization != bool(expected_outputs):
+        raise ValueError(
+            "expected_outputs must list the candidate charts unless options.expects_visualization is false, "
+            f"got {len(expected_outputs)} candidate(s) with expects_visualization={expects_visualization}"
+        )
     client = ChatClient(
         host=host,
         token=token,
@@ -343,7 +381,9 @@ def run_agentic_visualization(
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
             run_results.append(
-                _execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations, requires_execution)
+                _execute_single_run(
+                    client, conv_id_0, question, expected_outputs, max_iterations, requires_execution, options
+                )
             )
         finally:
             if initial_conversation_id is None:
@@ -353,7 +393,9 @@ def run_agentic_visualization(
             conv_id = client.create_conversation()
             try:
                 run_results.append(
-                    _execute_single_run(client, conv_id, question, expected_outputs, max_iterations, requires_execution)
+                    _execute_single_run(
+                        client, conv_id, question, expected_outputs, max_iterations, requires_execution, options
+                    )
                 )
             finally:
                 client.delete_conversation(conv_id)
@@ -393,6 +435,54 @@ def _run_detail(run: RunResult) -> dict:
     }
 
 
+def _strict_checks(ev: EvaluationResult) -> dict[str, bool]:
+    """The scores one run is gated on, by their Langfuse names.
+
+    An item that expects no chart is scored on that and on its answer only: the chart checks
+    would all read False and say nothing about it.
+    """
+    if not ev.expects_visualization or (ev.chart_optional and not ev.visualization_created):
+        checks = {"assertion-vis-no-chart": not ev.visualization_created}
+    else:
+        checks = {
+            "assertion-cross-ref-valid": ev.cross_ref_valid,
+            "assertion-vis-metric": ev.metrics_correct,
+            "assertion-vis-dimensions": ev.dimensions_correct,
+            "assertion-vis-filters": ev.filters_correct,
+            "assertion-vis-type": ev.viz_type_hard,
+        }
+        if ev.requires_execution:
+            checks["assertion-vis-executed"] = ev.executed
+        if ev.buckets_correct is not None:
+            checks["assertion-vis-buckets"] = ev.buckets_correct
+        if ev.format_correct is not None:
+            checks["assertion-vis-format"] = ev.format_correct
+        if ev.part_present is not None:
+            checks["assertion-vis-part-present"] = ev.part_present
+    if ev.answer_correct is not None:
+        checks["assertion-vis-answer"] = ev.answer_correct
+    return checks
+
+
+def _options_lines(ev: EvaluationResult, answer_text: str) -> str:
+    """The item-level verdicts for the failure summary; only the ones the item asks for."""
+    lines = ""
+    # A chart_optional item that built a chart is scored as a chart item, so "no chart" is not a check there.
+    if not ev.expects_visualization or (ev.chart_optional and not ev.visualization_created):
+        lines += f"  No Chart              : {not ev.visualization_created}\n"
+    if ev.answer_correct is not None:
+        lines += f"  Answer Correct        : {ev.answer_correct}\n"
+        lines += "".join(f"    {problem}\n" for problem in ev.answer_problems)
+        if ev.answer_problems:
+            lines += f"    answer   : {answer_text[:500]!r}\n"
+    if ev.format_correct is not None:
+        lines += f"  Format Correct        : {ev.format_correct}\n"
+        lines += "".join(f"    {problem}\n" for problem in ev.format_problems)
+    if ev.part_present is not None:
+        lines += f"  Part Present          : {ev.part_present}\n"
+    return lines
+
+
 def _buckets_line(ev: EvaluationResult) -> str:
     """The bucket-by-bucket verdict for the failure summary, or "" when the item does not ask for it."""
     lines = "".join(f"    {problem}\n" for problem in ev.metric_layout_problems)
@@ -437,6 +527,7 @@ def evaluate_agentic_visualization(
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     user_context: dict | None = None,
     requires_execution: bool = False,
+    options: ExpectationOptions | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.
@@ -472,6 +563,7 @@ def evaluate_agentic_visualization(
             agent_id=agent_id,
             user_context=user_context,
             requires_execution=requires_execution,
+            options=options,
         )
 
     if langfuse is not None and dataset_item_id:
@@ -485,25 +577,12 @@ def _write_scores(ctx: RunTraceContext) -> None:
             for run_idx, run in enumerate(summary.run_results):
                 pt = ctx.trace(run.conversation_id)
                 ev = run.eval_result
-                strict_checks = {
-                    "assertion-cross-ref-valid": ev.cross_ref_valid,
-                    "assertion-vis-metric": ev.metrics_correct,
-                    "assertion-vis-dimensions": ev.dimensions_correct,
-                    "assertion-vis-filters": ev.filters_correct,
-                    "assertion-vis-type": ev.viz_type_hard,
-                }
-                if ev.requires_execution:
-                    strict_checks["assertion-vis-executed"] = ev.executed
-                if ev.buckets_correct is not None:
-                    strict_checks["assertion-vis-buckets"] = ev.buckets_correct
+                strict_checks = _strict_checks(ev)
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
-                    ctx.score(tid, name="assertion-cross-ref-valid", value=ev.cross_ref_valid, data_type="BOOLEAN")
-                    ctx.score(tid, name="assertion-vis-metric", value=ev.metrics_correct, data_type="BOOLEAN")
-                    ctx.score(tid, name="assertion-vis-dimensions", value=ev.dimensions_correct, data_type="BOOLEAN")
-                    ctx.score(tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN")
-                    ctx.score(tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN")
-                    if ev.buckets_correct is not None:
-                        ctx.score(tid, name="assertion-vis-buckets", value=ev.buckets_correct, data_type="BOOLEAN")
+                    for score_name, value in strict_checks.items():
+                        # Scored below on every item, gating or not.
+                        if score_name != "assertion-vis-executed":
+                            ctx.score(tid, name=score_name, value=value, data_type="BOOLEAN")
                     ctx.score(tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN")
                     # Scored on every item, gating or not, so the run rate is visible per model.
                     ctx.score(tid, name="assertion-vis-executed", value=ev.executed, data_type="BOOLEAN")
@@ -572,7 +651,11 @@ def _write_scores(ctx: RunTraceContext) -> None:
         n = len(expected_outputs)
         candidate_note = f" (closest of {n} candidates)" if n > 1 else ""
         cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else ""
-        expected_dump = best.best_expected.model_dump(exclude_none=True)
+        expected_dump = (
+            best.best_expected.model_dump(exclude_none=True)
+            if best.best_expected is not None
+            else {"expects_visualization": False}
+        )
         actual_dump = best.actual_output.model_dump(exclude_none=True) if best.actual_output else None
         exc = VisualizationAssertionError(
             "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
@@ -604,6 +687,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
             f"  Viz Type Hard         : {ev.viz_type_hard}\n"
             f"{_buckets_line(ev)}"
             f"  Executed              : {ev.executed}{' (required)' if ev.requires_execution else ''}\n"
+            f"{_options_lines(ev, best.answer_text)}"
             f"  Stated Value Matches  : {ev.stated_value_matches} (reported only)\n"
             "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
         )
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py
index d184e6d6d..360a366f6 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py
@@ -2,10 +2,13 @@
 """Agentic visualization evaluator — ported from gdc-nas tavern-e2e app/vis_agentic.py."""
 
 import re
+from collections.abc import Mapping
 from dataclasses import dataclass, field, replace
 from datetime import date
 from typing import Any
 
+from gooddata_eval.core.chat.render import render_answer_text
+from gooddata_eval.core.chat.sse_client import _ADHOC_VIZ_ID
 from gooddata_eval.core.evaluators.base import ItemEvaluation
 from gooddata_eval.core.models import (
     ChatResult,
@@ -14,6 +17,7 @@
     ToolCallEvent,
     timeline_detail,
 )
+from gooddata_eval.core.number_format import check_format_intents, validate_format_intents
 from gooddata_eval.core.scoring import (
     bucket_layout,
     check_buckets,
@@ -71,9 +75,21 @@ class EvaluationResult:
     actual_buckets: dict[str, list[str]] | None = None
     # Why `metrics_correct` is False although the metric set matched: order or axis.
     metric_layout_problems: list[str] = field(default_factory=list)
+    # The item-level checks of `ExpectationOptions`. None means the item does not ask for one.
+    expects_visualization: bool = True
+    chart_optional: bool = False
+    answer_correct: bool | None = None
+    answer_problems: list[str] = field(default_factory=list)
+    format_correct: bool | None = None
+    format_problems: list[str] = field(default_factory=list)
+    part_present: bool | None = None
 
     @property
     def strict_pass(self) -> bool:
+        if not self.expects_visualization:
+            return not self.visualization_created and self.answer_correct is not False
+        if self.chart_optional and not self.visualization_created:
+            return self.answer_correct is not False
         return (
             self.visualization_created
             and self.cross_ref_valid
@@ -84,6 +100,9 @@ def strict_pass(self) -> bool:
             and self.viz_type_hard
             and (self.executed or not self.requires_execution)
             and self.buckets_correct is not False
+            and self.answer_correct is not False
+            and self.format_correct is not False
+            and self.part_present is not False
         )
 
     @property
@@ -97,6 +116,9 @@ def strict_checks_passed_count(self) -> int:
                 self.sorts_correct,
                 self.viz_type_hard,
                 self.buckets_correct is True,
+                self.answer_correct is True,
+                self.format_correct is True,
+                self.part_present is True,
             ]
         )
 
@@ -223,6 +245,144 @@ def requires_execution_of(expected_output: object) -> bool:
     return isinstance(expected_output, dict) and expected_output.get("requires_execution") is True
 
 
+@dataclass(frozen=True)
+class ExpectationOptions:
+    """The item-level keys of a visualization expectation, beside its candidate charts.
+
+    Every key is opt-in: the defaults score an item exactly as before they existed.
+    - ``expects_visualization`` False: the agent must answer without a chart, for example by
+      asking for the part the request leaves out. The run then stops after one turn, since a
+      simulated reply would push the agent into building the chart.
+    - ``chart_optional``: either outcome is right -- an answer without a chart, scored on the
+      answer alone, or a chart, scored against the candidates and the answer. For a request
+      the chart type cannot draw as asked, where asking back and building the closest chart
+      with an explanation are both correct. The run stops after one turn, as above.
+    - ``answer_must_include`` / ``answer_must_not_match``: regular expressions, matched without
+      regard to case against everything the final turn said. Every ``must_include`` pattern has
+      to match and no ``must_not_match`` pattern may.
+    - ``format_intents``: per metric uri, how its numbers must read (see ``number_format``).
+    - ``require_visualization_part``: the chart must come back as a part of the answer. Without
+      it, a chart that exists only in the tool arguments is scored like one the user saw.
+    """
+
+    requires_execution: bool = False
+    expects_visualization: bool = True
+    chart_optional: bool = False
+    answer_must_include: tuple[str, ...] = ()
+    answer_must_not_match: tuple[str, ...] = ()
+    format_intents: Mapping[str, Mapping[str, Any] | None] | None = None
+    require_visualization_part: bool = False
+
+
+def _regex_error(pattern: str) -> str | None:
+    try:
+        re.compile(pattern)
+    except re.error as exc:
+        return str(exc)
+    return None
+
+
+def _patterns(expected_output: dict, key: str) -> tuple[str, ...]:
+    patterns = expected_output.get(key, [])
+    if not isinstance(patterns, list) or not all(isinstance(p, str) and p for p in patterns):
+        raise ValueError(f"{key} must be a list of regular expressions, got {patterns!r}")
+    for pattern in patterns:
+        error = _regex_error(pattern)
+        if error is not None:
+            raise ValueError(f"{key} holds an invalid regular expression {pattern!r}: {error}")
+    return tuple(patterns)
+
+
+def expectation_options(expected_output: object) -> ExpectationOptions:
+    """Read and validate the item-level keys of a visualization expectation.
+
+    Raises:
+        ValueError: a key is malformed. Loud, before any request: a typo would otherwise
+            assert nothing and read green.
+    """
+    if not isinstance(expected_output, dict):
+        return ExpectationOptions()
+    expects = expected_output.get("expects_visualization", True)
+    if not isinstance(expects, bool):
+        raise ValueError(f"expects_visualization must be true or false, got {expects!r}")
+    optional = expected_output.get("chart_optional", False)
+    if not isinstance(optional, bool):
+        raise ValueError(f"chart_optional must be true or false, got {optional!r}")
+    if optional and not expects:
+        raise ValueError("chart_optional and expects_visualization false contradict each other")
+    # Checked here because every path reads the options: a candidate parser that only drops
+    # the candidates would score a contradictory item as a no-chart item.
+    if not expects and any(expected_output.get(key) is not None for key in ("visualization", "expected_outputs")):
+        raise ValueError("an expectation without a chart must not list 'visualization' candidates")
+    part = expected_output.get("require_visualization_part", False)
+    if not isinstance(part, bool):
+        raise ValueError(f"require_visualization_part must be true or false, got {part!r}")
+    intents = expected_output.get("format_intents")
+    if intents is not None:
+        validate_format_intents(intents)
+    if not expects and (intents is not None or part):
+        raise ValueError("an expectation without a chart cannot state format_intents or require_visualization_part")
+    return ExpectationOptions(
+        requires_execution=requires_execution_of(expected_output),
+        expects_visualization=expects,
+        chart_optional=optional,
+        answer_must_include=_patterns(expected_output, "answer_must_include"),
+        answer_must_not_match=_patterns(expected_output, "answer_must_not_match"),
+        format_intents=intents,
+        require_visualization_part=part,
+    )
+
+
+def check_answer(text: str, must_include: tuple[str, ...], must_not_match: tuple[str, ...]) -> list[str]:
+    """What the answer text gets wrong against the patterns; [] when it gets nothing wrong.
+
+    A typographic apostrophe is read as a plain one, so "aren’t" matches a pattern written
+    with "aren't".
+    """
+    text = text.replace("\u2019", "'")
+    problems = [f"the answer does not match {p!r}" for p in must_include if not re.search(p, text, re.IGNORECASE)]
+    for pattern in must_not_match:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if match:
+            problems.append(f"the answer matches {pattern!r}: {match.group(0)!r}")
+    return problems
+
+
+def with_options(
+    ev: EvaluationResult,
+    options: ExpectationOptions,
+    actual: CreatedVisualization | None,
+    tool_call_events: list[ToolCallEvent],
+    answer_text: str,
+) -> EvaluationResult:
+    """``ev`` with the item-level checks of ``options`` applied."""
+    asks_answer = bool(options.answer_must_include or options.answer_must_not_match)
+    answer_problems = (
+        check_answer(answer_text, options.answer_must_include, options.answer_must_not_match) if asks_answer else []
+    )
+    # An item that accepts no chart still needs an answer. A failed chat request and a silent
+    # turn both end with neither, and would otherwise pass as a correct "no chart".
+    accepts_no_chart = not options.expects_visualization or options.chart_optional
+    silent = accepts_no_chart and actual is None and not answer_text.strip()
+    if silent:
+        answer_problems = [*answer_problems, "the agent returned neither a chart nor an answer"]
+    checks_answer = asks_answer or silent
+    format_problems = check_format_intents(options.format_intents, tool_call_events) if options.format_intents else []
+    return replace(
+        ev,
+        expects_visualization=options.expects_visualization,
+        chart_optional=options.chart_optional,
+        answer_correct=not answer_problems if checks_answer else None,
+        answer_problems=answer_problems,
+        format_correct=not format_problems if options.format_intents else None,
+        format_problems=format_problems,
+        # The fallback chart, built from the tool arguments when the answer carried no part, has this id.
+        part_present=(actual is not None and actual.id != _ADHOC_VIZ_ID)
+        if options.require_visualization_part
+        else None,
+    )
+
+
 def _check_visualization_skill_activated(tool_call_events: list[ToolCallEvent]) -> bool:
     """Return True if set_skills was called with 'visualization' in skill_names."""
     for tc in tool_call_events:
@@ -305,12 +465,41 @@ def _evaluate_visualization(
     )
 
 
+def _no_candidate_result(actual: CreatedVisualization | None, skill_activated: bool) -> EvaluationResult:
+    """The result for an item that expects no chart: only whether one was built counts."""
+    return EvaluationResult(
+        visualization_created=actual is not None,
+        cross_ref_valid=False,
+        metrics_correct=False,
+        dimensions_correct=False,
+        filters_correct=False,
+        sorts_correct=False,
+        viz_type_hard=False,
+        filter_date_score=False,
+        filter_ranking_score=False,
+        filter_attribute_score=False,
+        skill_activated=skill_activated,
+        cross_ref_errors=[],
+        expected_metric_uris=set(),
+        actual_metric_uris=get_metric_uri_set(actual) if actual else set(),
+        expected_dim_uris=set(),
+        actual_dim_uris=get_dimension_uri_set(actual) if actual else set(),
+        expected_filters={category: [] for category in _NO_FILTERS},
+        actual_filters={category: [] for category in _NO_FILTERS},
+        expected_sorts=[],
+        actual_sorts=[],
+        expects_visualization=False,
+    )
+
+
 def _evaluate_against_candidates(
     expected_outputs: list[CreatedVisualization],
     actual: CreatedVisualization | None,
     skill_activated: bool = False,
     today: date | None = None,
-) -> tuple[EvaluationResult, CreatedVisualization]:
+) -> tuple[EvaluationResult, CreatedVisualization | None]:
+    if not expected_outputs:
+        return _no_candidate_result(actual, skill_activated), None
     # Anchored once here too: the candidates are ranked against each other, so letting
     # each resolve its own today could rank them on spans from different days.
     today = today or date.today()
@@ -323,6 +512,9 @@ def _parse_expected(expected_output: dict) -> list[CreatedVisualization]:
     if not isinstance(expected_output, dict):
         raise ValueError("'expected_output' must be a JSON object")
     raw_viz = expected_output.get("visualization")
+    if expected_output.get("expects_visualization") is False:
+        # `expectation_options`, read first, has rejected candidates listed beside the flag.
+        return []
     if raw_viz is None:
         raise ValueError("'expected_output.visualization' is required")
     if isinstance(raw_viz, list):
@@ -347,6 +539,18 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict:
     Shared by the single-shot evaluator below and the agentic path
     (``core/agentic/visualization.py``) so both report the exact same shape.
     """
+    answer = {
+        **({"answer_correct": ev.answer_correct} if ev.answer_correct is not None else {}),
+        **({"answer_problems": ev.answer_problems} if ev.answer_problems else {}),
+    }
+    if not ev.expects_visualization or (ev.chart_optional and not ev.visualization_created):
+        return {
+            "no_chart": not ev.visualization_created,
+            **answer,
+            "skill_activated": ev.skill_activated,
+            "actual_metric_uris": sorted(ev.actual_metric_uris),
+            "actual_dim_uris": sorted(ev.actual_dim_uris),
+        }
     return {
         "visualization_created": ev.visualization_created,
         "cross_ref_valid": ev.cross_ref_valid,
@@ -385,6 +589,10 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict:
             "stated_value_matches": ev.stated_value_matches,
         },
         **({"executed": ev.executed} if ev.requires_execution else {}),
+        **answer,
+        **({"format_correct": ev.format_correct} if ev.format_correct is not None else {}),
+        **({"format_problems": ev.format_problems} if ev.format_problems else {}),
+        **({"part_present": ev.part_present} if ev.part_present is not None else {}),
     }
 
 
@@ -392,13 +600,13 @@ class VisualizationEvaluator:
     test_kind = "visualization"
 
     def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
+        options = expectation_options(item.expected_output)
         candidates = _parse_expected(item.expected_output)
         actual = _extract_actual(chat_result)
         skill_activated = _check_visualization_skill_activated(chat_result.tool_call_events)
         ev, _best_expected = _evaluate_against_candidates(candidates, actual, skill_activated)
-        ev = with_execution(
-            ev, chat_result.tool_call_events, chat_result.text_response, requires_execution_of(item.expected_output)
-        )
+        ev = with_execution(ev, chat_result.tool_call_events, chat_result.text_response, options.requires_execution)
+        ev = with_options(ev, options, actual, chat_result.tool_call_events, render_answer_text(chat_result))
         return ItemEvaluation(
             passed=ev.strict_pass,
             rank_key=(ev.strict_pass, ev.strict_checks_passed_count),
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/number_format.py b/packages/gooddata-eval/src/gooddata_eval/core/number_format.py
new file mode 100644
index 000000000..f03ee3e44
--- /dev/null
+++ b/packages/gooddata-eval/src/gooddata_eval/core/number_format.py
@@ -0,0 +1,177 @@
+# (C) 2026 GoodData Corporation
+"""Whether the number format an agent wrote for a metric does what the user asked for.
+
+The agent writes a `format` and a `format_example` (`{value, shown_as}`) for each formatted
+metric, and the chart tool rejects the call unless the format renders `value` as
+`shown_as`. So `shown_as` is how the chart really reads `value`. What the tool cannot know
+is whether that reading is what the user asked for: a "millions" request answered with
+`#,##0.0M` and the example `4,221,301,265.5M` passes the tool and shows a number a million
+times too large (GDAI-2592). This module reads the example back against the user's intent.
+
+An intent is one of:
+- `{"divide_by": 1000000, "decimals": 1, "suffix": "M", "prefix": ""}`: value / divide_by;
+- `{"kind": "auto_scale", "decimals": 1}`: B, M or K by the size of the value;
+- `{"kind": "percent", "decimals": 1}`: a ratio times 100, with "%";
+- `{"negative": "parentheses", "decimals": 0}`: a negative value written in parentheses;
+- `None`: no format may be written at all.
+
+The shown text is compared as a number, not as a string, because equally correct formats
+differ in grouping and spacing: `#,##0.0,,M` reads `4,221.3M` and `#,,.0 M` reads `4221.3 M`.
+"""
+
+import re
+from collections.abc import Mapping
+from typing import Any
+
+from gooddata_eval.core.models import ToolCallEvent
+from gooddata_eval.core.scoring import resolve_alias_to_uri
+
+_INTENT_KEYS = frozenset({"kind", "divide_by", "decimals", "prefix", "suffix", "negative"})
+_KINDS = frozenset({"auto_scale", "percent"})
+_TIERS = ((1e9, "B"), (1e6, "M"), (1e3, "K"))
+_NUMBER = re.compile(r"\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?")
+
+
+def validate_format_intents(intents: object) -> None:
+    """Reject a malformed `format_intents` before any request.
+
+    Raises:
+        ValueError: the mapping or one of its intents is unusable.
+    """
+    if not isinstance(intents, Mapping) or not intents:
+        raise ValueError(f"format_intents must map metric uris to an intent or null, got {intents!r}")
+    for uri, intent in intents.items():
+        if not isinstance(uri, str) or not uri.startswith(("metric/", "fact/")):
+            raise ValueError(f"format_intents keys must be metric/ or fact/ uris, got {uri!r}")
+        if intent is None:
+            continue
+        if not isinstance(intent, Mapping):
+            raise ValueError(f"format_intents[{uri!r}] must be an object or null, got {intent!r}")
+        unknown = sorted(set(intent) - _INTENT_KEYS)
+        if unknown:
+            raise ValueError(f"format_intents[{uri!r}] states {unknown}, which is not one of {sorted(_INTENT_KEYS)}")
+        if intent.get("kind") is not None and intent["kind"] not in _KINDS:
+            raise ValueError(f"format_intents[{uri!r}] kind must be one of {sorted(_KINDS)}, got {intent['kind']!r}")
+        if intent.get("negative") not in (None, "parentheses"):
+            raise ValueError(f"format_intents[{uri!r}] negative must be 'parentheses', got {intent['negative']!r}")
+        decimals = intent.get("decimals", 0)
+        if not isinstance(decimals, int) or isinstance(decimals, bool) or decimals < 0:
+            raise ValueError(f"format_intents[{uri!r}] decimals must be a non-negative integer, got {decimals!r}")
+        divide_by = intent.get("divide_by", 1)
+        if not isinstance(divide_by, (int, float)) or isinstance(divide_by, bool) or divide_by <= 0:
+            raise ValueError(f"format_intents[{uri!r}] divide_by must be a positive number, got {divide_by!r}")
+
+
+def _authored_metric_entries(tool_call_events: list[ToolCallEvent]) -> dict[str, dict]:
+    """The metric entries of the last successful `create_adhoc_visualization` call, by metric uri.
+
+    `format_example` exists only in the tool arguments: the tool drops it from the chart it
+    returns, so the chart part of the answer cannot be read for it. A metric sits in `metrics`,
+    in a bubble chart's `size_by`, or in a repeater's `columns`, as the scorer counts it.
+    """
+    entries: dict[str, dict] = {}
+    for tc in tool_call_events:
+        if tc.function_name != "create_adhoc_visualization":
+            continue
+        result = tc.parsed_result()
+        if not isinstance(result, dict) or result.get("status", "success") != "success":
+            continue
+        viz = tc.parsed_arguments().get("visualization")
+        if not isinstance(viz, dict):
+            continue
+        fields = (viz.get("query") or {}).get("fields") or {}
+        entries = {}
+        for entry in [*(viz.get("metrics") or []), *(viz.get("size_by") or []), *(viz.get("columns") or [])]:
+            if isinstance(entry, dict) and isinstance(entry.get("field"), str):
+                entries[resolve_alias_to_uri(entry["field"], fields)] = entry
+    return entries
+
+
+def _target(intent: Mapping[str, Any], value: float) -> tuple[float, str, str, bool] | str:
+    """`(number shown, prefix, suffix, parentheses)` the intent asks for, or why the example cannot tell."""
+    divide_by = intent.get("divide_by", 1)
+    prefix, suffix = intent.get("prefix", ""), intent.get("suffix", "")
+    if intent.get("negative") == "parentheses":
+        if value >= 0:
+            return f"the example value {value} is not negative, so it cannot show the parentheses"
+        return abs(value) / divide_by, prefix, suffix, True
+    if intent.get("kind") == "percent":
+        return value * 100, prefix, "%", False
+    if intent.get("kind") == "auto_scale":
+        for threshold, tier in _TIERS:
+            if abs(value) >= threshold:
+                return value / threshold, prefix, tier, False
+        return f"the example value {value} is below a thousand, so it cannot show a B/M/K tier"
+    if abs(value) < divide_by:
+        return f"the example value {value} is below {divide_by}, so it cannot show the scaling"
+    return value / divide_by, prefix, suffix, False
+
+
+def _read(shown: str, prefix: str, suffix: str, parentheses: bool) -> tuple[float, int] | None:
+    """The number `shown` states and its decimal places, or None when it is not written that way."""
+    text = "".join(shown.split())
+    negative = False
+    if parentheses:
+        if not (text.startswith("(") and text.endswith(")")):
+            return None
+        text, negative = text[1:-1], True
+    if text.startswith("-"):
+        text, negative = text[1:], True
+    if prefix and not text.startswith(prefix):
+        return None
+    text = text[len(prefix) :]
+    if text.startswith("-"):
+        text, negative = text[1:], True
+    if suffix:
+        if not text.endswith(suffix):
+            return None
+        text = text[: -len(suffix)]
+    if not _NUMBER.fullmatch(text):
+        return None
+    number = float(text.replace(",", ""))
+    decimals = len(text.split(".", 1)[1]) if "." in text else 0
+    return (-number if negative else number), decimals
+
+
+def format_problem(intent: Mapping[str, Any], example: object) -> str | None:
+    """Why the example does not read the way the intent asks, or None when it does."""
+    if not isinstance(example, Mapping):
+        return "no format_example was written"
+    value, shown = example.get("value"), example.get("shown_as")
+    if not isinstance(value, (int, float)) or isinstance(value, bool) or not isinstance(shown, str):
+        return f"the format_example {dict(example)!r} has no numeric value to check"
+    target = _target(intent, float(value))
+    if isinstance(target, str):
+        return target
+    number, prefix, suffix, parentheses = target
+    decimals = intent.get("decimals", 0)
+    read = _read(shown, prefix, suffix, parentheses)
+    expected_text = f"{'(' if parentheses else ''}{prefix}{number:,.{decimals}f}{suffix}{')' if parentheses else ''}"
+    if read is None:
+        return f"{value} is shown as {shown!r}, expected {expected_text!r}"
+    shown_number, shown_decimals = read
+    wanted = -number if parentheses else number
+    if shown_decimals != decimals or abs(shown_number - wanted) > 0.5 * 10**-decimals + 1e-9:
+        return f"{value} is shown as {shown!r}, expected {expected_text!r}"
+    return None
+
+
+def check_format_intents(
+    intents: Mapping[str, Mapping[str, Any] | None], tool_call_events: list[ToolCallEvent]
+) -> list[str]:
+    """Every metric with an intent must carry a format whose example reads as asked; [] when all do."""
+    entries = _authored_metric_entries(tool_call_events)
+    problems: list[str] = []
+    for uri, intent in intents.items():
+        entry = entries.get(uri)
+        if intent is None:
+            if entry is not None and entry.get("format"):
+                problems.append(f"{uri}: a format {entry['format']!r} was written, but none was asked for")
+            continue
+        if entry is None or not entry.get("format"):
+            problems.append(f"{uri}: no format was written")
+            continue
+        problem = format_problem(intent, entry.get("format_example"))
+        if problem is not None:
+            problems.append(f"{uri}: {problem} (format {entry['format']!r})")
+    return problems
diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py
index 01bec5e12..f28c0fae6 100644
--- a/packages/gooddata-eval/tests/test_agentic_runner.py
+++ b/packages/gooddata-eval/tests/test_agentic_runner.py
@@ -135,6 +135,42 @@ def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_
     assert mock_eval.call_args.kwargs["agent_id"] == "agent-1"
 
 
+def _dispatch_visualization(expected_output: dict) -> Any:
+    item = DatasetItem(
+        id="q1", dataset_name="ds", test_kind="vis_agentic", question="q", expected_output=expected_output
+    )
+    with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_visualization") as mock_eval:
+        _dispatch_agentic(
+            item,
+            host="https://h",
+            token="tok",
+            workspace_id="ws1",
+            k=1,
+            langfuse=None,
+            run_ts="2026-01-01",
+            model_version_override=None,
+        )
+    return mock_eval.call_args.kwargs
+
+
+def test_dispatch_agentic_relays_an_item_without_a_chart_with_no_candidates():
+    kwargs = _dispatch_visualization({"expects_visualization": False, "answer_must_include": ["destination"]})
+    assert kwargs["expected_outputs"] == []
+    assert kwargs["options"].expects_visualization is False
+    assert kwargs["options"].answer_must_include == ("destination",)
+
+
+@pytest.mark.parametrize(
+    "candidates",
+    [{"visualization": _MIN_VIZ}, {"expected_outputs": [{"visualization": _MIN_VIZ}]}],
+    ids=["visualization", "expected_outputs"],
+)
+def test_dispatch_agentic_rejects_an_item_without_a_chart_that_lists_candidates(candidates):
+    """Dropping the candidates would score the contradiction as a no-chart item and read green."""
+    with pytest.raises(ValueError, match="must not list 'visualization' candidates"):
+        _dispatch_visualization({"expects_visualization": False, **candidates})
+
+
 def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem:
     return DatasetItem(
         id="item-1",
@@ -839,7 +875,14 @@ class _Stop(Exception):
 # Kinds whose run_* hands the item's context to its ChatClient, which sends it on every turn.
 # conversation passes it per turn instead, covered in its own test file.
 _CLIENT_CONTEXT_KINDS = [
-    ("visualization", "run_agentic_visualization", "evaluate_agentic_visualization", []),
+    # An empty candidate list is rejected before the client exists: it is only right for an item
+    # that expects no chart.
+    (
+        "visualization",
+        "run_agentic_visualization",
+        "evaluate_agentic_visualization",
+        [CreatedVisualization.model_validate(_MIN_VIZ)],
+    ),
     ("metric_skill", "run_agentic_metric_skill", "evaluate_agentic_metric_skill", {}),
     (
         "dashboard_skill",
@@ -1026,10 +1069,9 @@ def close(self) -> None:
         pass
 
 
-_RUN_ZERO_KINDS = [
-    (m, run_fn, expected or [CreatedVisualization.model_validate(_MIN_VIZ)] if m == "visualization" else expected)
-    for m, run_fn, _, expected in _CLIENT_CONTEXT_KINDS
-] + [("conversation", "run_agentic_conversation", _MIN_CONVERSATION_FIXTURE)]
+_RUN_ZERO_KINDS = [(m, run_fn, expected) for m, run_fn, _, expected in _CLIENT_CONTEXT_KINDS] + [
+    ("conversation", "run_agentic_conversation", _MIN_CONVERSATION_FIXTURE)
+]
 
 
 @pytest.mark.parametrize(("module_name", "run_fn", "expected"), _RUN_ZERO_KINDS)
diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py
index 499f274bf..d196f0f7e 100644
--- a/packages/gooddata-eval/tests/test_agentic_visualization.py
+++ b/packages/gooddata-eval/tests/test_agentic_visualization.py
@@ -4,16 +4,20 @@
 All tests mock ChatClient so no network is needed.
 """
 
+from dataclasses import replace
 from unittest.mock import MagicMock, call, patch
 
 import pytest
+from gooddata_eval.core.agentic import visualization as agentic_visualization
 from gooddata_eval.core.agentic.visualization import (
     VisualizationAssertionError,
     _execute_single_run,
+    _strict_checks,
     evaluate_agentic_visualization,
     run_agentic_visualization,
 )
 from gooddata_eval.core.chat.sse_client import ChatError
+from gooddata_eval.core.evaluators.visualization import ExpectationOptions
 from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit
 
 
@@ -550,3 +554,218 @@ def _boom(*_a, **_k):
 
     assert result.exit_reason is LoopExit.SIMULATED_USER_FAILED
     assert result.eval_result.visualization_created is False
+
+
+def test_an_item_without_a_chart_stops_after_the_first_answer(monkeypatch):
+    """A simulated reply would answer the agent's question and push it into the chart the item rules out."""
+    replies = MagicMock(side_effect=AssertionError("no simulated reply may be sent"))
+    monkeypatch.setattr(agentic_visualization, "generate_simulated_response", replies)
+    client = MagicMock()
+    client.send_message.return_value = _chat_clarification("Which destination should the sankey flow to?")
+    options = ExpectationOptions(expects_visualization=False, answer_must_include=("destination",))
+
+    result = _execute_single_run(client, "conv-1", "Create a sankey from Region", [], max_iterations=4, options=options)
+
+    assert client.send_message.call_count == 1
+    assert result.exit_reason is LoopExit.SUCCESS
+    assert result.best_expected is None
+    assert result.eval_result.strict_pass
+    assert _strict_checks(result.eval_result) == {"assertion-vis-no-chart": True, "assertion-vis-answer": True}
+
+
+def test_strict_checks_publish_only_the_options_an_item_asks_for():
+    client = MagicMock()
+    client.send_message.return_value = _chat_with_viz()
+    plain = _execute_single_run(client, "conv-1", "q", [_expected()], max_iterations=1)
+    assert set(_strict_checks(plain.eval_result)) == {
+        "assertion-cross-ref-valid",
+        "assertion-vis-metric",
+        "assertion-vis-dimensions",
+        "assertion-vis-filters",
+        "assertion-vis-type",
+    }
+    with_options = replace(
+        plain.eval_result,
+        requires_execution=True,
+        buckets_correct=True,
+        format_correct=False,
+        part_present=True,
+        answer_correct=True,
+    )
+    assert set(_strict_checks(with_options)) - set(_strict_checks(plain.eval_result)) == {
+        "assertion-vis-executed",
+        "assertion-vis-buckets",
+        "assertion-vis-format",
+        "assertion-vis-part-present",
+        "assertion-vis-answer",
+    }
+
+
+# ── an item that accepts no chart still needs an answer ─────────────────────
+
+
+def _silent_turn() -> MagicMock:
+    client = MagicMock()
+    client.send_message.return_value = ChatResult.model_validate({"toolCallEvents": [], "reasoningStepCount": 0})
+    return client
+
+
+def _failed_request() -> MagicMock:
+    client = MagicMock()
+    client.send_message.side_effect = ChatError("gen-ai unavailable")
+    return client
+
+
+@pytest.mark.parametrize(
+    ("make_client", "exit_reason"),
+    [(_silent_turn, LoopExit.AGENT_SILENT), (_failed_request, LoopExit.CHAT_ERROR)],
+    ids=["silent", "chat_error"],
+)
+@pytest.mark.parametrize(
+    ("options", "expected_outputs"),
+    [
+        (ExpectationOptions(expects_visualization=False), []),
+        (ExpectationOptions(expects_visualization=False, answer_must_not_match=("sankey",)), []),
+        (ExpectationOptions(chart_optional=True), [_expected()]),
+    ],
+    ids=["no_chart", "no_chart_must_not_match", "chart_optional"],
+)
+def test_a_run_with_neither_a_chart_nor_an_answer_fails_an_item_that_accepts_no_chart(
+    make_client, exit_reason, options, expected_outputs
+):
+    """With no answer pattern, or only a must-not-match one, an empty run would read as a correct "no chart"."""
+    result = _execute_single_run(make_client(), "conv-1", "q", expected_outputs, options=options)
+
+    assert result.exit_reason is exit_reason
+    assert result.eval_result.strict_pass is False
+    assert result.eval_result.answer_correct is False
+    assert result.eval_result.answer_problems[-1] == "the agent returned neither a chart nor an answer"
+    assert _strict_checks(result.eval_result)["assertion-vis-answer"] is False
+
+
+def test_an_answer_without_a_pattern_passes_an_item_without_a_chart():
+    client = MagicMock()
+    client.send_message.return_value = _chat_clarification("Which destination?")
+
+    result = _execute_single_run(client, "conv-1", "q", [], options=ExpectationOptions(expects_visualization=False))
+
+    assert result.eval_result.strict_pass is True
+    assert result.eval_result.answer_correct is None
+
+
+@pytest.mark.parametrize(
+    ("expected_outputs", "options"),
+    [([], None), ([], ExpectationOptions()), ([_expected()], ExpectationOptions(expects_visualization=False))],
+    ids=["no_candidates_no_options", "no_candidates_default_options", "candidates_but_no_chart"],
+)
+def test_run_rejects_candidates_that_disagree_with_the_options_before_any_request(expected_outputs, options):
+    with (
+        patch.object(agentic_visualization, "ChatClient") as mock_client,
+        pytest.raises(ValueError, match="expected_outputs must list the candidate charts"),
+    ):
+        run_agentic_visualization("https://h", "tok", "ws", "q", expected_outputs, options=options)
+    mock_client.assert_not_called()
+
+
+# ── the failure summary and the scores show the options an item asks for ───
+
+
+def test_a_failed_item_reports_and_scores_every_option_it_asks_for():
+    expected = CreatedVisualization.model_validate({**_viz(), "bucket_strict": True})
+    # The same fields, with the quarter moved from view_by to segment_by: only the bucket check sees it.
+    moved = {**_viz(), "view_by": [], "segment_by": ["d"]}
+    client = MagicMock()
+    client.create_conversation.return_value = "conv-1"
+    client.send_message.return_value = ChatResult.model_validate(
+        {
+            "createdVisualizations": {"objects": [moved], "reasoning": ""},
+            "textResponse": "Here it is.",
+            "toolCallEvents": [],
+        }
+    )
+    options = ExpectationOptions(
+        requires_execution=True,
+        answer_must_include=("revenue",),
+        format_intents={"metric/revenue": {"divide_by": 1000, "suffix": "K"}},
+        require_visualization_part=True,
+    )
+    captured = {}
+
+    def _capture(_submit, _identity, **kwargs):
+        captured["write_scores"] = kwargs["write_scores"]
+
+    with (
+        patch.object(agentic_visualization, "ChatClient", return_value=client),
+        patch.object(agentic_visualization, "submit_trace_scoring", _capture),
+        pytest.raises(VisualizationAssertionError) as exc_info,
+    ):
+        evaluate_agentic_visualization(
+            host="https://example.com",
+            token="tok",
+            workspace_id="ws",
+            question="Show revenue",
+            expected_outputs=[expected],
+            k=1,
+            options=options,
+            langfuse=MagicMock(),
+            dataset_item_id="item-1",
+        )
+
+    message = str(exc_info.value)
+    assert "Buckets Correct       : False" in message
+    assert "expected : {'view_by': ['label/date.quarter']" in message
+    assert "Answer Correct        : False" in message
+    assert "the answer does not match 'revenue'" in message
+    assert "answer   : 'Here it is.'" in message
+    assert "Format Correct        : False" in message
+    assert "metric/revenue: no format was written" in message
+    assert "Part Present          : True" in message
+    assert "No Chart" not in message
+
+    ctx = MagicMock()
+    captured["write_scores"](ctx)
+    scores = [(c.kwargs["name"], c.kwargs["value"]) for c in ctx.score.call_args_list]
+    names = [name for name, _ in scores]
+    assert dict(scores)["assertion-vis-buckets"] is False
+    assert dict(scores)["assertion-vis-format"] is False
+    assert dict(scores)["assertion-vis-part-present"] is True
+    assert dict(scores)["assertion-vis-answer"] is False
+    # Gating here, but still written once: the loop leaves it to the every-item score.
+    assert names.count("assertion-vis-executed") == 1
+
+
+def test_a_failed_item_without_a_chart_reports_the_chart_it_built():
+    client = MagicMock()
+    client.create_conversation.return_value = "conv-1"
+    client.send_message.return_value = _chat_with_viz()
+
+    with (
+        patch.object(agentic_visualization, "ChatClient", return_value=client),
+        pytest.raises(VisualizationAssertionError) as exc_info,
+    ):
+        evaluate_agentic_visualization(
+            host="https://example.com",
+            token="tok",
+            workspace_id="ws",
+            question="Create a sankey from Region",
+            expected_outputs=[],
+            k=1,
+            options=ExpectationOptions(expects_visualization=False),
+        )
+
+    message = str(exc_info.value)
+    assert "No Chart              : False" in message
+    assert '"expects_visualization": false' in message
+
+
+def test_a_chart_optional_item_that_built_a_chart_does_not_report_a_no_chart_line():
+    """A chart is a valid outcome there, so a "No Chart : False" line would read as a failed check."""
+    client = MagicMock()
+    client.send_message.return_value = _chat_with_viz()
+    options = ExpectationOptions(chart_optional=True)
+    built = _execute_single_run(client, "conv-1", "q", [_expected()], options=options)
+    assert "No Chart" not in agentic_visualization._options_lines(built.eval_result, built.answer_text)
+
+    client.send_message.return_value = _chat_clarification("Which measure should the donut show?")
+    asked = _execute_single_run(client, "conv-1", "q", [_expected()], options=options)
+    assert "No Chart              : True" in agentic_visualization._options_lines(asked.eval_result, asked.answer_text)
diff --git a/packages/gooddata-eval/tests/test_number_format.py b/packages/gooddata-eval/tests/test_number_format.py
new file mode 100644
index 000000000..1213500ff
--- /dev/null
+++ b/packages/gooddata-eval/tests/test_number_format.py
@@ -0,0 +1,174 @@
+# (C) 2026 GoodData Corporation
+"""Number-format intents (GDAI-2592). The shown texts are the ones GoodData's formatter
+produced for the patterns the assistant wrote on stg11, 2026-10-13."""
+
+import json
+
+import pytest
+from gooddata_eval.core.models import ToolCallEvent
+from gooddata_eval.core.number_format import check_format_intents, format_problem, validate_format_intents
+
+_MILLIONS = {"divide_by": 1000000, "decimals": 1, "suffix": "M"}
+
+
+@pytest.mark.parametrize(
+    ("intent", "value", "shown"),
+    [
+        (_MILLIONS, 4221301265.5, "4,221.3M"),  # #,##0.0,,M
+        (_MILLIONS, 4221301265.5, "4221.3 M"),  # #,,.0 M -- no grouping, a space: still millions
+        ({"divide_by": 1000, "decimals": 1, "suffix": "K"}, 24096.1, "24.1K"),
+        ({"kind": "auto_scale", "decimals": 1}, 4221301265.5, "4.2 B"),
+        ({"kind": "auto_scale", "decimals": 1}, 24096.1, "24.1 K"),
+        ({"kind": "percent", "decimals": 1}, 0.253, "25.3%"),
+        ({"prefix": "$", "decimals": 0}, 1234.4, "$1,234"),
+        ({"negative": "parentheses", "decimals": 0}, -1234.5, "(1,235)"),
+        ({"negative": "parentheses", "decimals": 0}, -1234.5, "(1,234)"),  # either rounding of .5
+    ],
+)
+def test_an_example_that_reads_as_asked_passes(intent, value, shown):
+    assert format_problem(intent, {"value": value, "shown_as": shown}) is None
+
+
+@pytest.mark.parametrize(
+    ("intent", "value", "shown"),
+    [
+        (_MILLIONS, 4221301265.5, "4,221,301,265.5M"),  # #,##0.0M: a letter appended, nothing divided
+        (_MILLIONS, 4221301265.5, "4,221M"),  # millions, but no decimal
+        ({"kind": "auto_scale", "decimals": 1}, 4221301265.5, "4,221,301,265.5a"),  # numeral.js syntax
+        ({"kind": "percent", "decimals": 1}, 0.253, "0.3%"),
+        ({"negative": "parentheses", "decimals": 0}, -1234.5, "-1,235"),
+    ],
+)
+def test_an_example_that_does_not_read_as_asked_fails(intent, value, shown):
+    problem = format_problem(intent, {"value": value, "shown_as": shown})
+    assert problem is not None and "expected" in problem
+
+
+@pytest.mark.parametrize(
+    ("intent", "value"),
+    [
+        (_MILLIONS, 12.0),  # below a million: a format that divides nothing reads the same
+        ({"kind": "auto_scale", "decimals": 1}, 999.0),
+        ({"negative": "parentheses", "decimals": 0}, 5.0),
+    ],
+)
+def test_an_example_that_cannot_show_the_intent_fails(intent, value):
+    assert "cannot show" in format_problem(intent, {"value": value, "shown_as": str(value)})
+
+
+def test_a_missing_or_null_example_fails():
+    assert format_problem(_MILLIONS, None) == "no format_example was written"
+    assert "no numeric value" in format_problem(_MILLIONS, {"value": None, "shown_as": "N/A"})
+
+
+def _create_call(metrics: list, status: str = "success") -> ToolCallEvent:
+    args = {
+        "visualization": {
+            "type": "bar_chart",
+            "query": {"fields": {"m": {"using": "metric/net_sales"}, "d": {"using": "label/product_category"}}},
+            "metrics": metrics,
+            "view_by": ["d"],
+        }
+    }
+    return ToolCallEvent.model_validate(
+        {
+            "functionName": "create_adhoc_visualization",
+            "functionArguments": json.dumps(args),
+            "result": json.dumps({"status": status, "ref": "viz_1"}),
+        }
+    )
+
+
+def test_the_last_successful_chart_call_is_the_one_read():
+    wrong = _create_call(
+        [
+            {
+                "field": "m",
+                "format": "#,##0.0M",
+                "format_example": {"value": 4221301265.5, "shown_as": "4,221,301,265.5M"},
+            }
+        ]
+    )
+    right = _create_call(
+        [{"field": "m", "format": "#,##0.0,,M", "format_example": {"value": 4221301265.5, "shown_as": "4,221.3M"}}]
+    )
+    rejected = _create_call([{"field": "m", "format": "#,##0.0M"}], status="error")
+    intents = {"metric/net_sales": _MILLIONS}
+    assert check_format_intents(intents, [wrong, right, rejected]) == []
+    problems = check_format_intents(intents, [right, wrong])
+    assert problems and "4,221,301,265.5M" in problems[0]
+
+
+def test_a_metric_without_a_format_fails_and_one_that_needs_none_must_carry_none():
+    plain = _create_call(["m"])
+    assert check_format_intents({"metric/net_sales": _MILLIONS}, [plain]) == ["metric/net_sales: no format was written"]
+    assert check_format_intents({"metric/net_sales": None}, [plain]) == []
+    formatted = _create_call([{"field": "m", "format": "$#,##0", "format_example": {"value": 1, "shown_as": "$1"}}])
+    assert "none was asked for" in check_format_intents({"metric/net_sales": None}, [formatted])[0]
+
+
+@pytest.mark.parametrize(
+    ("intents", "message"),
+    [
+        ({}, "must map metric uris"),
+        ({"label/x": None}, "metric/ or fact/ uris"),
+        ({"metric/x": {"divide": 1000}}, "which is not one of"),
+        ({"metric/x": {"kind": "scaled"}}, "kind must be one of"),
+        ({"metric/x": {"decimals": -1}}, "decimals must be"),
+        ({"metric/x": {"divide_by": 0}}, "divide_by must be"),
+        ({"metric/x": {"negative": "red"}}, "negative must be"),
+    ],
+)
+def test_a_malformed_intent_is_rejected(intents, message):
+    with pytest.raises(ValueError, match=message):
+        validate_format_intents(intents)
+
+
+def test_a_null_intent_is_valid_and_a_scalar_one_is_rejected():
+    validate_format_intents({"metric/x": None, "metric/y": _MILLIONS})
+    with pytest.raises(ValueError, match="must be an object or null"):
+        validate_format_intents({"metric/x": "millions"})
+
+
+@pytest.mark.parametrize(
+    ("intent", "value", "shown", "ok"),
+    [
+        ({"prefix": "$", "decimals": 0}, -1234.4, "-$1,234", True),  # sign before the prefix
+        ({"prefix": "$", "decimals": 0}, -1234.4, "$-1,234", True),  # sign after the prefix
+        ({"prefix": "$", "decimals": 0}, 1234.4, "1,234", False),  # the prefix is missing
+        ({"decimals": 0}, 1234.4, "1,23,4", False),  # not a number GoodData writes
+        ({"decimals": 0}, 1234.4, "about 1,234", False),
+    ],
+)
+def test_the_sign_the_prefix_and_the_digits_of_the_shown_text_are_read(intent, value, shown, ok):
+    assert (format_problem(intent, {"value": value, "shown_as": shown}) is None) is ok
+
+
+def _call(function_name: str, arguments: dict) -> ToolCallEvent:
+    return ToolCallEvent.model_validate(
+        {
+            "functionName": function_name,
+            "functionArguments": json.dumps(arguments),
+            "result": json.dumps({"status": "success", "ref": "viz_1"}),
+        }
+    )
+
+
+def test_other_tool_calls_and_a_call_without_a_chart_are_not_read():
+    formatted = _create_call(
+        [{"field": "m", "format": "#,##0.0,,M", "format_example": {"value": 4221301265.5, "shown_as": "4,221.3M"}}]
+    )
+    other = _call("execute_visualization", {"visualization_ref": "viz_1"})
+    no_chart = _call("create_adhoc_visualization", {"visualization": "viz_1"})
+    assert check_format_intents({"metric/net_sales": _MILLIONS}, [formatted, other, no_chart]) == []
+
+
+@pytest.mark.parametrize("bucket", ["size_by", "columns"])
+def test_a_format_on_a_metric_outside_the_metrics_bucket_is_read(bucket):
+    """A bubble chart sizes by a metric in `size_by`; a repeater writes its metrics into `columns`."""
+    entry = {"field": "m", "format": "#,##0.0,,M", "format_example": {"value": 4221301265.5, "shown_as": "4,221.3M"}}
+    call = _create_call([])
+    arguments = json.loads(call.function_arguments)
+    arguments["visualization"][bucket] = [entry]
+    call = _call("create_adhoc_visualization", arguments)
+    assert check_format_intents({"metric/net_sales": _MILLIONS}, [call]) == []
diff --git a/packages/gooddata-eval/tests/test_visualization_evaluator.py b/packages/gooddata-eval/tests/test_visualization_evaluator.py
index ae37b4722..9fad06011 100644
--- a/packages/gooddata-eval/tests/test_visualization_evaluator.py
+++ b/packages/gooddata-eval/tests/test_visualization_evaluator.py
@@ -1,10 +1,17 @@
 # (C) 2026 GoodData Corporation
+import json
 import re
 from datetime import date
 
 import pytest
+from gooddata_eval.core.chat.sse_client import _ADHOC_VIZ_ID
 from gooddata_eval.core.evaluators import get_evaluator
-from gooddata_eval.core.evaluators.visualization import _evaluate_visualization, execution_signals
+from gooddata_eval.core.evaluators.visualization import (
+    ExpectationOptions,
+    _evaluate_visualization,
+    execution_signals,
+    expectation_options,
+)
 from gooddata_eval.core.models import ChatResult, CreatedVisualization, DatasetItem, ToolCallEvent
 from gooddata_eval.core.scoring import get_dimension_uri_set, get_metric_uri_set
 
@@ -544,3 +551,171 @@ def test_expectation_switches_are_left_out_of_a_recorded_output():
     dumped = viz.model_dump(exclude_none=True)
     assert "bucket_strict" not in dumped
     assert "metrics_ordered" not in dumped
+
+
+# -- Item-level options (QA-29722) ------------------------------------------------------------
+
+
+def _item_with(expected_output: dict) -> DatasetItem:
+    return DatasetItem(
+        id="i1", dataset_name="d", test_kind="visualization", question="q", expected_output=expected_output
+    )
+
+
+def test_an_item_without_a_chart_passes_on_a_question_and_fails_on_a_chart():
+    ev = get_evaluator("visualization")
+    expected = {"expects_visualization": False, "answer_must_include": ["destination"]}
+    asked = ChatResult.model_validate(
+        {"textResponse": "A Sankey chart needs a source and a destination. Which destination?", "toolCallEvents": []}
+    )
+    result = ev.evaluate(_item_with(expected), asked)
+    assert result.passed, result.detail
+    assert result.detail["no_chart"] is True
+    assert "metrics_correct" not in result.detail
+    built = ev.evaluate(_item_with(expected), _chat_result_with(_expected()))
+    assert not built.passed
+    assert built.detail["no_chart"] is False
+
+
+def test_a_no_chart_expectation_that_lists_charts_is_rejected():
+    with pytest.raises(ValueError, match="must not list 'visualization' candidates"):
+        get_evaluator("visualization").evaluate(
+            _item_with({"expects_visualization": False, "visualization": _expected()}), _chat_result_with(_expected())
+        )
+
+
+def test_answer_patterns_ignore_case_and_read_a_typographic_apostrophe():
+    ev = get_evaluator("visualization")
+    reply = {
+        "createdVisualizations": {"objects": [_expected()], "reasoning": ""},
+        "textResponse": "Gantt charts aren’t supported here, so I created the closest option.",
+        "toolCallEvents": [],
+    }
+    included = ev.evaluate(
+        _item_with({"visualization": _expected(), "answer_must_include": ["AREN'T SUPPORTED"]}),
+        ChatResult.model_validate(reply),
+    )
+    assert included.passed, included.detail
+    forbidden = ev.evaluate(
+        _item_with({"visualization": _expected(), "answer_must_not_match": ["n't supported"]}),
+        ChatResult.model_validate(reply),
+    )
+    assert not forbidden.passed
+    assert forbidden.detail["answer_correct"] is False
+    assert "n't supported" in forbidden.detail["answer_problems"][0]
+
+
+def test_an_invalid_answer_pattern_is_rejected():
+    with pytest.raises(ValueError, match="invalid regular expression"):
+        get_evaluator("visualization").evaluate(
+            _item_with({"visualization": _expected(), "answer_must_include": ["(unclosed"]}),
+            _chat_result_with(_expected()),
+        )
+
+
+def test_a_chart_only_in_the_tool_arguments_fails_when_the_part_is_required():
+    ev = get_evaluator("visualization")
+    expected = {"visualization": _expected(), "require_visualization_part": True}
+    shown = ev.evaluate(_item_with(expected), _chat_result_with(_expected()))
+    assert shown.passed and shown.detail["part_present"] is True
+    fallback = ev.evaluate(_item_with(expected), _chat_result_with({**_expected(), "id": _ADHOC_VIZ_ID}))
+    assert not fallback.passed and fallback.detail["part_present"] is False
+    unasked = ev.evaluate(_item(_expected()), _chat_result_with({**_expected(), "id": _ADHOC_VIZ_ID}))
+    assert unasked.passed and "part_present" not in unasked.detail
+
+
+def test_a_format_intent_is_read_from_the_chart_tool_call():
+    ev = get_evaluator("visualization")
+    args = {
+        "visualization": {
+            **_expected(),
+            "metrics": [
+                {
+                    "field": "m_rev",
+                    "format": "#,##0.0M",
+                    "format_example": {"value": 4221301265.5, "shown_as": "4,221,301,265.5M"},
+                }
+            ],
+        }
+    }
+    chat = ChatResult.model_validate(
+        {
+            "createdVisualizations": {"objects": [_expected()], "reasoning": ""},
+            "toolCallEvents": [
+                {
+                    "functionName": "create_adhoc_visualization",
+                    "functionArguments": json.dumps(args),
+                    "result": json.dumps({"status": "success", "ref": "viz_1"}),
+                }
+            ],
+        }
+    )
+    expected = {
+        "visualization": _expected(),
+        "format_intents": {"metric/revenue": {"divide_by": 1000000, "decimals": 1, "suffix": "M"}},
+    }
+    result = ev.evaluate(_item_with(expected), chat)
+    assert not result.passed
+    assert result.detail["format_correct"] is False
+    assert "4,221,301,265.5M" in result.detail["format_problems"][0]
+
+
+def test_an_optional_chart_passes_on_either_an_answer_or_the_right_chart():
+    ev = get_evaluator("visualization")
+    expected = {"visualization": _expected(), "chart_optional": True, "answer_must_include": ["cannot|which"]}
+    asked = ChatResult.model_validate({"textResponse": "Which measure should the donut show?", "toolCallEvents": []})
+    by_answer = ev.evaluate(_item_with(expected), asked)
+    assert by_answer.passed and by_answer.detail["no_chart"] is True
+    built = {
+        "createdVisualizations": {"objects": [_expected()], "reasoning": ""},
+        "textResponse": "A donut cannot show two measures, so I created a column chart.",
+        "toolCallEvents": [],
+    }
+    by_chart = ev.evaluate(_item_with(expected), ChatResult.model_validate(built))
+    assert by_chart.passed and by_chart.detail["metrics_correct"] is True
+    wrong_chart = {**built, "createdVisualizations": {"objects": [{**_expected(), "type": "line_chart"}]}}
+    assert not ev.evaluate(_item_with(expected), ChatResult.model_validate(wrong_chart)).passed
+
+
+def test_an_optional_chart_cannot_also_rule_the_chart_out():
+    with pytest.raises(ValueError, match="contradict"):
+        get_evaluator("visualization").evaluate(
+            _item_with({"expects_visualization": False, "chart_optional": True}), _chat_result_with(_expected())
+        )
+
+
+@pytest.mark.parametrize(
+    ("expected_output", "message"),
+    [
+        ({"visualization": _expected(), "answer_must_include": "destination"}, "must be a list of regular"),
+        ({"visualization": _expected(), "answer_must_not_match": [""]}, "must be a list of regular"),
+        ({"expects_visualization": "no"}, "expects_visualization must be true or false"),
+        ({"visualization": _expected(), "chart_optional": 1}, "chart_optional must be true or false"),
+        ({"visualization": _expected(), "require_visualization_part": "yes"}, "require_visualization_part must be"),
+        ({"expects_visualization": False, "require_visualization_part": True}, "cannot state format_intents"),
+        (
+            {"expects_visualization": False, "format_intents": {"metric/revenue": None}},
+            "cannot state format_intents",
+        ),
+        ({"expects_visualization": False, "expected_outputs": []}, "must not list 'visualization' candidates"),
+    ],
+)
+def test_a_malformed_expectation_option_is_rejected(expected_output, message):
+    with pytest.raises(ValueError, match=message):
+        expectation_options(expected_output)
+
+
+def test_an_expectation_that_is_not_an_object_reads_the_defaults():
+    assert expectation_options([{"visualization": _expected()}]) == ExpectationOptions()
+
+
+@pytest.mark.parametrize(
+    "expected_output", [{"expects_visualization": False}, {"visualization": _expected(), "chart_optional": True}]
+)
+def test_an_empty_turn_fails_an_item_that_accepts_no_chart(expected_output):
+    result = get_evaluator("visualization").evaluate(
+        _item_with(expected_output), ChatResult.model_validate({"toolCallEvents": []})
+    )
+    assert not result.passed
+    assert result.detail["answer_correct"] is False
+    assert result.detail["answer_problems"] == ["the agent returned neither a chart nor an answer"]