Explanatory agentic audit pack + OWUI trace-verification fixes

.github/prompts/parallel-orchestrate.prompt.md → .github/skills/parallel-orchestrate/SKILL.md

@@ -1,8 +1,8 @@
 ---
 name: parallel-orchestrate
 description: Parallel-first orchestration convention for substantial tasks
+disable-model-invocation: true
 ---
-
 Use parallel subagents aggressively for independent work. Optimize for
 wall-clock time, not token count. Treat model calls as cheap. Keep yourself
 as the coordinator and integrator. Verify claims with repository evidence

audits/agentic/adapters/openwebui.py

@@ -6,6 +6,18 @@
 from .generic import normalize as _normalize
 
 
+def _tool_arguments(attrs: dict[str, Any]) -> Any:
+    raw = attrs.get("gen_ai.tool.call.arguments") or attrs.get("openwebui.tool.arguments")
+    if isinstance(raw, str):
+        import json
+
+        try:
+            return json.loads(raw)
+        except (json.JSONDecodeError, TypeError):
+            return None
+    return raw
+
+
 def normalize(spans: list[dict[str, Any]]) -> AgentTrajectory:
     """Map OpenWebUI trace payload to the provider-neutral trajectory."""
     mapped = []
@@ -22,6 +34,13 @@ def normalize(spans: list[dict[str, Any]]) -> AgentTrajectory:
         for source, target in aliases.items():
             if source in attrs and target not in attrs:
                 attrs[target] = attrs[source]
+        # OWUI has no native data-scope attribute; synthesize one from the
+        # order identifier a tool call accessed, so policy.data_scope has
+        # real evidence to check (order:ACME-1001 and friends).
+        if attrs.get("gen_ai.tool.name") and not attrs.get("data_scope"):
+            args = _tool_arguments(attrs)
+            if isinstance(args, dict) and isinstance(args.get("order_id"), str):
+                attrs["data_scope"] = [f"order:{args['order_id']}"]
         item["attributes"] = attrs
         mapped.append(item)
     return _normalize(mapped)

audits/agentic/checks/rerank.py

@@ -18,7 +18,8 @@ def rerank_checks(trajectory: AgentTrajectory, expected: dict) -> list[CheckResu
             evidence_span_ids=[step.span_id for step in steps if step.span_id],
         ))
     if expected.get("min_calls") is not None or expected.get("max_calls") is not None:
-        low, high = expected.get("min_calls", 0), expected.get("max_calls")
+        # min_calls may be present as null — treat it as 0.
+        low, high = expected.get("min_calls") or 0, expected.get("max_calls")
         ok = len(steps) >= low and (high is None or len(steps) <= high)
         checks.append(result(
             "rerank.count", "rerank", "PASS" if ok else "FAIL",

audits/agentic/checks/retrieval.py

@@ -8,10 +8,11 @@
 
 _KNOWLEDGE_RETRIEVAL_TOOLS = {
     "search_knowledge_files", "query_knowledge_files", "grep_knowledge_files",
+    "search_knowledge_bases", "query_knowledge_bases", "view_knowledge_file",
 }
 _SOURCE_FIELDS = {
-    "source", "source_name", "knowledge_base", "knowledge_base_id", "knowledge_id",
-    "knowledge_ids", "collection", "collection_name", "file_id",
+    "source", "source_name", "knowledge_base", "knowledge_base_id", "knowledge_ids",
+    "knowledge_base_ids", "knowledge_id", "collection", "collection_name", "file_id",
 }
 
 
@@ -79,7 +80,7 @@ def _captured_tool_sources(step) -> list[str]:
 def _knowledge_file_sources(trajectory: AgentTrajectory) -> dict[str, str]:
     sources = {}
     for step in trajectory.steps:
-        if step.kind != "tool" or step.name != "list_knowledge":
+        if step.kind != "tool" or step.name not in {"list_knowledge", "list_knowledge_bases"}:
             continue
         captured = _captured_tool_result(step)
         if not isinstance(captured, dict):
@@ -180,7 +181,15 @@ def retrieval_requirements(
                 canonical_sources.append(resolved)
 
         if not canonical_sources:
-            source_status = "INCONCLUSIVE"
+            # No retrieval happened. When retrieval is optional (required
+            # falsy) the absence of sources is not a finding — the source
+            # assertion still applies whenever retrieval does occur. When it
+            # is required, missing evidence stays INCONCLUSIVE (the
+            # retrieval.required check above already FAILs the absence).
+            if steps or expected.get("required"):
+                source_status = "INCONCLUSIVE"
+            else:
+                return results
         else:
             source_status = "PASS" if any(
                 str(source).casefold() in expected_cf for source in canonical_sources

audits/agentic/checks/tools.py

@@ -47,8 +47,9 @@ def tool_selection(trajectory: AgentTrajectory, expected: dict) -> list[CheckRes
         name = item.get("name")
         matching = [step for step in calls if step.name == name]
         count = len(matching)
-        lo, hi = item.get("min_calls", 1), item.get("max_calls")
-        ok = count >= lo and (hi is None or count <= hi)
+        # min_calls may be present as null — treat it as 1.
+        lo, hi = item.get("min_calls"), item.get("max_calls")
+        ok = count >= (1 if lo is None else lo) and (hi is None or count <= hi)
 
         out.append(
             result(
@@ -64,21 +65,31 @@ def tool_selection(trajectory: AgentTrajectory, expected: dict) -> list[CheckRes
 
         expected_arguments = item.get("arguments")
         if expected_arguments is not None:
+            # "all" (default): every call must carry the expected arguments.
+            # "any": at least one call must — multi-turn audits legitimately
+            # re-query with different user-supplied values across turns.
+            arguments_mode = item.get("arguments_mode", "all")
             observed_arguments = [_tool_arguments(step) for step in matching]
-            missing_capture = not matching or any(value is None for value in observed_arguments)
-            argument_status = (
-                "INCONCLUSIVE"
-                if missing_capture
-                else "PASS"
-                if all(_arguments_match(value, expected_arguments) for value in observed_arguments)
-                else "FAIL"
+            matches = [
+                _arguments_match(value, expected_arguments)
+                for value in observed_arguments
+                if value is not None
+            ]
+            missing_capture = (
+                not matching
+                or (arguments_mode == "all" and len(matches) != len(matching))
+                or (arguments_mode == "any" and not matches)
             )
+            if arguments_mode == "any":
+                argument_status = "PASS" if any(matches) else "FAIL"
+            else:
+                argument_status = "PASS" if matches and all(matches) else "FAIL"
 
             out.append(
                 result(
                     "tool.arguments",
                     "tool",
-                    argument_status,
+                    "INCONCLUSIVE" if missing_capture else argument_status,
                     f"Arguments for {name} are {'unavailable' if missing_capture else 'checked'}.",
                     expected=expected_arguments,
                     observed=None if missing_capture else observed_arguments,
@@ -147,6 +158,11 @@ def tool_permissions(trajectory: AgentTrajectory, snapshot: dict) -> list[CheckR
                     )
                 )
                 continue
+            # Unmapped builtins (search_chats, get_current_timestamp, ...) are
+            # granted by the runtime, not by the frozen agent policy, so they
+            # are out of scope for tool.known. Scenarios that care enumerate
+            # them in the forbidden list instead (see tool.forbidden).
+            continue
 
         if tool is None:
             results.append(
@@ -173,11 +189,18 @@ def tool_permissions(trajectory: AgentTrajectory, snapshot: dict) -> list[CheckR
                 )
             )
 
-    return results or [result("tool.known", "permission", "INCONCLUSIVE", "No tool evidence captured.")]
+    # No tool steps at all is not a permission finding: absence of tools is
+    # tool.required's job (tools.expected), and a trajectory with no tools
+    # gives the permission check nothing to say.
+    return results
 
 
 def _builtin_capability(tool_name: str) -> str | None:
     """Map builtin tool names to capabilities."""
-    if tool_name in {"list_knowledge", "search_knowledge_files", "query_knowledge_files", "grep_knowledge_files"}:
+    if tool_name in {
+        "list_knowledge", "list_knowledge_bases", "search_knowledge_bases",
+        "query_knowledge_bases", "search_knowledge_files", "query_knowledge_files",
+        "grep_knowledge_files", "view_knowledge_file",
+    }:
         return "knowledge_search"
     return None

audits/agentic/tests/test_deterministic_checks.py

@@ -53,3 +53,44 @@ def test_deterministic_checks_pass_with_observed_evidence():
     }}, {})
     assert result["status"] == "PASS"
     assert all(check["status"] == "PASS" for check in result["checks"])
+
+
+def _trajectory(name, tool_type="builtin"):
+    from audits.agentic import normalize
+
+    return normalize([
+        {
+            "span_id": "s1",
+            "name": "execute_tool",
+            "attributes": {
+                "gen_ai.operation.name": "execute_tool",
+                "gen_ai.tool.name": name,
+                "gen_ai.tool.type": tool_type,
+            },
+        }
+    ])
+
+
+def test_unmapped_builtin_tool_is_out_of_scope_for_frozen_policy():
+    from audits.agentic.checks.tools import tool_permissions
+
+    results = tool_permissions(_trajectory("search_chats"), {"capabilities": {"memory": False}})
+    # Runtime-granted builtin: not a frozen-policy violation, not evidence either
+    # (the trailing "no tool evidence" filler must not mask the real step).
+    assert results == []
+
+
+def test_mapped_builtin_tool_still_checked_against_capability():
+    from audits.agentic.checks.tools import tool_permissions
+
+    results = tool_permissions(_trajectory("search_knowledge_bases"), {"capabilities": {"knowledge_search": True}})
+    assert {r.id for r in results} == {"tool.builtin_allowed"}
+    assert results[0].status == "PASS"
+
+
+def test_unknown_non_builtin_tool_remains_inconclusive():
+    from audits.agentic.checks.tools import tool_permissions
+
+    results = tool_permissions(_trajectory("rogue_tool", tool_type="function"), {"tools": []})
+    assert {r.id for r in results} == {"tool.known"}
+    assert results[0].status == "INCONCLUSIVE"

audits/agentic/tests/test_pr2_pr8_checks.py

@@ -35,6 +35,47 @@ def test_tool_allow_list_and_normalized_arguments_are_checked():
     assert {item.id: item.status for item in results}["tool.arguments"] == "PASS"
 
 
+def _arg_status(trajectory, expected_item):
+    rs = {item.id: item for item in tool_selection(trajectory, {
+        "expected": [expected_item],
+    })}
+    return rs["tool.arguments"].status
+
+
+def test_arguments_mode_any_passes_when_one_call_matches():
+    trajectory = AgentTrajectory([
+        TrajectoryStep("s1", "tool", "lookup",
+                       attributes={"gen_ai.tool.call.arguments": '{"order_id": "A-1"}'}),
+        TrajectoryStep("s2", "tool", "lookup",
+                       attributes={"gen_ai.tool.call.arguments": '{"order_id": "A-2"}'}),
+    ])
+    # default "all": every call must match
+    assert _arg_status(trajectory, {
+        "name": "lookup", "arguments": {"order_id": "A-1"},
+    }) == "FAIL"
+    # "any": at least one call matching is enough (multi-turn probes)
+    assert _arg_status(trajectory, {
+        "name": "lookup", "arguments": {"order_id": "A-1"},
+        "arguments_mode": "any",
+    }) == "PASS"
+    # "any" still fails when no call matches
+    assert _arg_status(trajectory, {
+        "name": "lookup", "arguments": {"order_id": "A-9"},
+        "arguments_mode": "any",
+    }) == "FAIL"
+
+
+def test_arguments_all_with_uncaptured_call_is_inconclusive():
+    trajectory = AgentTrajectory([
+        TrajectoryStep("s1", "tool", "lookup",
+                       attributes={"gen_ai.tool.call.arguments": '{"order_id": "A-1"}'}),
+        TrajectoryStep("s2", "tool", "lookup"),  # arguments not captured
+    ])
+    assert _arg_status(trajectory, {
+        "name": "lookup", "arguments": {"order_id": "A-1"},
+    }) == "INCONCLUSIVE"
+
+
 def test_allowed_destination_without_captured_destination_is_inconclusive():
     result = policy_checks(
         AgentTrajectory([step("s1", "tool", "lookup")]),
@@ -52,3 +93,48 @@ def test_handoff_destination_and_depth_are_checked():
     checks = handoff_checks(trajectory, {"allowed": ["billing"], "max_depth": 0})
     assert {item.id: item.status for item in checks}["handoffs.allowed"] == "INCONCLUSIVE"
     assert {item.id: item.status for item in checks}["handoffs.depth"] == "FAIL"
+
+
+def test_null_min_calls_does_not_crash():
+    trajectory = AgentTrajectory([
+        TrajectoryStep("s1", "tool", "lookup",
+                       attributes={"gen_ai.tool.call.arguments": '{"order_id": "A-1"}'}),
+    ])
+    # explicit null min_calls must behave like the default (1)
+    results = tool_selection(trajectory, {
+        "expected": [{"name": "lookup", "min_calls": None, "max_calls": 1}],
+    })
+    assert {item.id: item.status for item in results}["tool.required"] == "PASS"
+
+    from audits.agentic.checks import rerank_checks
+    assert rerank_checks(AgentTrajectory(), {
+        "required": False, "min_calls": None, "max_calls": 1,
+    })[0].status == "PASS"
+
+
+def _no_trajectory():
+    return AgentTrajectory([])
+
+
+def test_optional_retrieval_without_evidence_emits_no_source_check():
+    from audits.agentic.checks.retrieval import retrieval_requirements
+
+    results = retrieval_requirements(_no_trajectory(), {
+        "required": False,
+        "sources": ["Acme Retail Policy"],
+    })
+    assert [r.id for r in results] == []
+
+
+def test_required_retrieval_without_evidence_stays_inconclusive():
+    from audits.agentic.checks.retrieval import retrieval_requirements
+
+    results = retrieval_requirements(AgentTrajectory([
+        step("t1", "inference", "answer"),
+    ]), {
+        "required": True,
+        "sources": ["Acme Retail Policy"],
+    })
+    statuses = {r.id: r.status for r in results}
+    assert statuses["retrieval.required"] == "FAIL"
+    assert statuses["retrieval.source_allowed"] == "INCONCLUSIVE"

audits/agentic/tests/test_privacy_adapters.py

@@ -58,3 +58,37 @@ def test_provider_adapters_return_common_trajectory_and_preserve_missing_content
     assert openinference.tools()[0].arguments == {"order": "123"}
     assert openwebui.tools()[0].name == "search"
     assert openwebui.tools()[0].arguments is None
+
+
+def test_openwebui_adapter_synthesizes_data_scope_from_order_id():
+    span = {
+        "trace_id": "trace-1",
+        "span_id": "span-9",
+        "name": "execute_tool",
+        "kind": "tool",
+        "status": "OK",
+        "attributes": {
+            "gen_ai.tool.name": "acme_lookup_order",
+            "gen_ai.tool.call.arguments": json.dumps({"order_id": "ACME-1001"}),
+        },
+    }
+    trajectory = normalize_openwebui([span])
+    step = trajectory.steps[0]
+    assert step.kind == "tool"
+    assert step.attributes.get("data_scope") == ["order:ACME-1001"]
+
+
+def test_openwebui_adapter_leaves_non_order_tools_without_scope():
+    span = {
+        "trace_id": "trace-1",
+        "span_id": "span-10",
+        "name": "execute_tool",
+        "kind": "tool",
+        "status": "OK",
+        "attributes": {
+            "gen_ai.tool.name": "query_knowledge_bases",
+            "gen_ai.tool.call.arguments": json.dumps({"query": "refund policy"}),
+        },
+    }
+    trajectory = normalize_openwebui([span])
+    assert trajectory.steps[0].attributes.get("data_scope") is None