Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions py/nox_session_constraints.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"test_livekit_agents": {"max_python_exclusive": "3.14", "ci_platforms": ["linux"]},
"test_pipecat": {"min_python": "3.11", "max_python_exclusive": "3.14"},
"test_agentscope": {"versions": {"latest": {"min_python": "3.11"}}},
"test_google_adk": {"versions": {"latest": {"min_python": "3.11"}}}, # google-adk 2.12 requires 3.11
"test_pydantic_ai_integration": {"versions": {"latest": {"min_python": "3.11"}}},
"test_pydantic_ai_logfire": {"versions": {"latest": {"min_python": "3.11"}}},
"test_pydantic_ai_wrap_openai": {"versions": {"latest": {"min_python": "3.11"}}},
Expand Down
17 changes: 5 additions & 12 deletions py/noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -463,19 +463,10 @@ def test_agno(session, version):
def test_livekit_agents(session, version):
_skip_if_incompatible(session)
_install_test_deps(session)
# Install the locked plugins first so the matrix pin decides livekit-agents;
# 1.3.x needs its own group to keep OpenTelemetry below LogData's removal.
_install_group_locked(session, "test-livekit-agents-1-3" if version == "1.3.1" else "test-livekit-agents")
_install_matrix_dep(session, "livekit-agents", version)
_install_group_locked(session, "test-livekit-agents")
if version == "1.3.1":
# LiveKit 1.3.1 accepts a newer OTLP exporter, but its dependencies
# require an SDK version that breaks LiveKit's LogData import. The
# locked group downgrades the exporter, so remove its now-orphaned deps.
session.run(
"uv",
"pip",
"uninstall",
"opentelemetry-exporter-otlp-common",
"opentelemetry-exporter-http-transport",
)
livekit_server_dir = _ensure_livekit_server(session)
env = {
"LIVEKIT_URL": os.environ.get("LIVEKIT_URL", "ws://localhost:7880"),
Expand Down Expand Up @@ -661,6 +652,7 @@ def test_crewai(session, version):
@nox.session()
@nox.parametrize("version", GOOGLE_ADK_VERSIONS, ids=GOOGLE_ADK_VERSIONS)
def test_google_adk(session, version):
_skip_if_incompatible(session)
_install_test_deps(session)
_install_matrix_dep(session, "google-adk", version, "test-sqlalchemy-2-0")
_run_tests(session, f"{INTEGRATION_DIR}/adk/test_adk.py", version=version)
Expand Down Expand Up @@ -925,6 +917,7 @@ def pylint(session):
minimum_python = {
"ai-sdk": (3, 12),
"agentscope": (3, 11),
"google-adk": (3, 11),
"harbor": (3, 12),
"pipecat-ai": (3, 11),
"pydantic-ai-integration": (3, 11),
Expand Down
74 changes: 48 additions & 26 deletions py/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@ test-litellm = [
# litellm requires openai<=1.99.9 (ResponseTextConfig removal in 1.100.0)
"openai==1.99.9",
"fastapi==0.135.3",
# litellm 1.106.0 imports mcp on every tool call but only declares it in
# the mcp/proxy extras (https://github.com/BerriAI/litellm/issues/46045).
"mcp>=2.2.0,<3",
"orjson==3.11.8; python_version<'3.15'",
# 3.11.8 has no cp315 wheels and its sdist fails on current stable Rust.
"orjson==3.12.0; python_version>='3.15'",
Expand Down Expand Up @@ -204,8 +207,18 @@ test-deepagents = [

test-livekit-agents = [
{include-group = "test"},
"livekit-plugins-openai",
"livekit-plugins-silero",
# Floors keep the lock on the matrix ``latest`` line; without them uv keeps
# older 1.3.x plugins, which pull livekit-agents back to 1.3.x.
"livekit-plugins-openai>=1.8.8",
"livekit-plugins-silero>=1.8.8",
]

test-livekit-agents-1-3 = [
{include-group = "test"},
# Pin the agents package so the plugins do not pull a newer 1.3.x.
"livekit-agents==1.3.1",
"livekit-plugins-openai==1.3.1",
"livekit-plugins-silero==1.3.1",
# livekit-agents 1.3.x imports opentelemetry.sdk._logs.LogData, removed in newer SDKs.
# This is a LiveKit runtime compatibility constraint, not a Braintrust OTel integration dependency.
"opentelemetry-sdk<1.39",
Expand Down Expand Up @@ -390,7 +403,7 @@ explicit = true
# ---------------------------------------------------------------------------

[tool.uv]
exclude-newer = "2026-10-15T00:00:00Z"
exclude-newer = "2026-10-19T06:03:00Z"
conflicts = [
# Groups that pin or transitively require conflicting openai versions.
[
Expand All @@ -406,17 +419,20 @@ conflicts = [
{group = "test-llamaindex-0-13"},
{group = "test-instructor"},
{group = "test-livekit-agents"},
{group = "test-livekit-agents-1-3"},
{group = "lint"},
],
# opentelemetry-sdk version conflicts (google-adk/livekit vs logfire).
[
{group = "lint"},
{group = "test-pydantic-ai-logfire"},
{group = "test-livekit-agents"},
{group = "test-livekit-agents-1-3"},
],
[
{group = "test-pydantic-ai-otel-events"},
{group = "test-livekit-agents"},
{group = "test-livekit-agents-1-3"},
],
[
{group = "lint"},
Expand All @@ -425,11 +441,17 @@ conflicts = [
[
{group = "test-pydantic-ai-logfire-constraints"},
{group = "test-livekit-agents"},
{group = "test-livekit-agents-1-3"},
],
[
{group = "test-llamaindex"},
{group = "test-llamaindex-0-13"},
],
# mcp 1.x constraints for old provider versions vs litellm's mcp 2.x.
[
{group = "test-litellm"},
{group = "test-mcp-v1"},
],
]

# ---------------------------------------------------------------------------
Expand All @@ -451,55 +473,55 @@ latest = "ai[openai]==0.8.0"
"0.4.0" = "ai[openai]==0.4.0"

[tool.braintrust.matrix.openai]
latest = "openai==3.29.0"
latest = "openai==3.35.1"
"1.92.0" = "openai==1.92.0"
"1.77.0" = "openai==1.77.0"
"1.71.0" = "openai==1.71.0"

[tool.braintrust.matrix.anthropic]
latest = "anthropic==1.13.0"
latest = "anthropic==1.15.0"
"0.48.0" = "anthropic==0.48.0"

[tool.braintrust.matrix.cohere]
latest = "cohere==7.2.0"
"5.0.0" = "cohere==5.0.0"

[tool.braintrust.matrix.openai-agents]
latest = "openai-agents==0.23.1"
latest = "openai-agents==0.24.0"
"0.13.6" = "openai-agents==0.13.6"
"0.0.19" = "openai-agents==0.0.19"

[tool.braintrust.matrix.litellm]
# 1.92.0 only publishes Linux wheels through CPython 3.13, but the test
# matrix also covers Windows and CPython 3.14.
latest = "litellm==1.105.0"
latest = "litellm==1.106.0"
"1.74.0" = "litellm==1.74.0"

[tool.braintrust.matrix.livekit-agents]
latest = "livekit-agents==1.8.6"
latest = "livekit-agents==1.8.8"
"1.3.1" = "livekit-agents==1.3.1"

[tool.braintrust.matrix.pipecat-ai]
latest = "pipecat-ai==1.12.0"
"1.3.0" = "pipecat-ai==1.3.0"

[tool.braintrust.matrix.claude-agent-sdk]
latest = "claude-agent-sdk==0.2.165"
latest = "claude-agent-sdk==0.2.168"
"0.1.10" = "claude-agent-sdk==0.1.10"

[tool.braintrust.matrix.cursor-sdk]
# 1.0.25 fixes bridge callback tokens that could be parsed as CLI flags while
# retaining per-turn usage, sync/async consumption, multimodal, and custom tools.
latest = "cursor-sdk==1.0.37"
latest = "cursor-sdk==1.0.38"
"1.0.25" = "cursor-sdk==1.0.25"

[tool.braintrust.matrix.agno]
latest = "agno==3.1.2"
latest = "agno==3.1.4"
"2.4.0" = "agno==2.4.0"
"2.1.0" = "agno==2.1.0"

[tool.braintrust.matrix.agentscope]
latest = "agentscope==2.0.9"
latest = "agentscope==2.0.10"
"1.0.0" = "agentscope==1.0.0"

[tool.braintrust.matrix.autogen-agentchat]
Expand All @@ -511,15 +533,15 @@ latest = "autogen-ext[openai]==0.7.5"
"0.7.0" = "autogen-ext[openai]==0.7.0"

[tool.braintrust.matrix.strands-agents]
latest = "strands-agents==1.59.0"
latest = "strands-agents==1.61.1"
"1.20.0" = "strands-agents==1.20.0"

[tool.braintrust.matrix.pydantic-ai-integration]
latest = "pydantic-ai==2.55.0"
latest = "pydantic-ai==2.57.0"
"1.10.0" = "pydantic-ai==1.10.0"

[tool.braintrust.matrix.pydantic-ai-wrap-openai]
latest = "pydantic-ai==2.55.0"
latest = "pydantic-ai==2.57.0"
"1.0.1" = "pydantic-ai==1.0.1"
"0.1.9" = "pydantic-ai==0.1.9"

Expand All @@ -532,43 +554,43 @@ latest = "instructor==1.17.0"
"1.11.0" = "instructor==1.11.0"

[tool.braintrust.matrix.google-genai]
latest = "google-genai==2.29.0"
latest = "google-genai==2.31.0"
"1.75.0" = "google-genai==1.75.0"
"1.30.0" = "google-genai==1.30.0"

[tool.braintrust.matrix.google-cloud-discoveryengine]
latest = "google-cloud-discoveryengine==0.20.5"

[tool.braintrust.matrix.dspy]
latest = "dspy==3.4.0"
latest = "dspy==3.4.1"
"2.6.0" = "dspy==2.6.0"

[tool.braintrust.matrix.deepagents]
latest = "deepagents==0.7.24"
latest = "deepagents==0.7.28"
"0.6.0" = "deepagents==0.6.0"

[tool.braintrust.matrix.crewai]
# 1.13.0 is the first release with the full causal-id surface (event_id /
# parent_event_id / started_event_id on BaseEvent) plus the ``usage`` field on
# LLMCallCompletedEvent that the Braintrust CrewAI integration depends on.
latest = "crewai==1.15.27"
latest = "crewai==1.15.28"
"1.13.0" = "crewai==1.13.0"

[tool.braintrust.matrix.google-adk]
latest = "google-adk[mcp]==2.11.0"
latest = "google-adk[mcp]==2.12.0"
"2.6.3" = "google-adk[mcp]==2.6.3"
"1.14.1" = "google-adk[mcp]==1.14.1"

[tool.braintrust.matrix.langchain-core]
latest = "langchain-core==1.6.9"
latest = "langchain-core==1.6.12"
"0.3.28" = "langchain-core==0.3.28"

[tool.braintrust.matrix.llama-index-core]
latest = "llama-index-core==0.14.25"
"0.13.0" = "llama-index-core==0.13.0"

[tool.braintrust.matrix.openrouter]
latest = "openrouter==1.3.43"
latest = "openrouter==1.3.66"
"0.6.0" = "openrouter==0.6.0"

[tool.braintrust.matrix.mistralai]
Expand All @@ -578,7 +600,7 @@ latest = "mistralai==3.2.0"

[tool.braintrust.matrix.typesafe-sdk]
latest = "typesafe-sdk==0.7.5"
"0.7.4" = "typesafe-sdk==0.7.4"
"0.6.0" = "typesafe-sdk==0.6.0"

[tool.braintrust.matrix.huggingface-hub]
# Floor pinned to 0.32.0: the earliest release that exposes the
Expand All @@ -589,7 +611,7 @@ latest = "huggingface-hub==2.2.0"
"0.32.0" = "huggingface-hub==0.32.0"

[tool.braintrust.matrix.transformers]
latest = "transformers==5.19.0"
latest = "transformers==5.20.1"
"4.42.0" = "transformers==4.42.0"

[tool.braintrust.matrix.temporalio]
Expand All @@ -615,11 +637,11 @@ latest = "pytest==9.1.1"
latest = "braintrust-core==0.0.59"

[tool.braintrust.matrix.boto3]
latest = "boto3==1.43.111"
latest = "boto3==1.43.114"
"1.34.116" = "boto3==1.34.116"

[tool.braintrust.matrix.botocore]
latest = "botocore==1.43.111"
latest = "botocore==1.43.114"
"1.34.116" = "botocore==1.34.116"

# OpenTelemetry api/sdk/exporter release in lockstep; keep their keys in sync.
Expand Down
58 changes: 40 additions & 18 deletions py/scripts/check-unused-cassettes.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,28 +161,50 @@ def sessions_by_integration(noxfile: pathlib.Path = _NOXFILE) -> dict[str, set[s
A session reads ``integrations/<name>/cassettes`` when it runs tests under
``integrations/<name>/``, or when it passes ``<name>`` as an env value to
``_run_tests`` (the btx sessions select their provider's cassettes that way).
Sessions are either functions that call ``_run_tests`` or spec dicts passed
to ``_register_matrix_sessions`` (with ``tests`` and optional ``env`` keys).
"""
known = set(integrations_with_cassettes())
mapping: dict[str, set[str]] = defaultdict(set)

def add(session: str, test_paths: list[ast.expr], env: ast.expr | None) -> None:
prefix = "braintrust/integrations/"
for node in test_paths:
path = _render_test_path(node)
if path and path.startswith(prefix):
name = path[len(prefix) :].split("/")[0]
if name in known:
mapping[name].add(session)
if isinstance(env, ast.Dict):
for value in env.values:
if isinstance(value, ast.Constant) and value.value in known:
mapping[value.value].add(session)

tree = ast.parse(noxfile.read_text())
for func in tree.body:
if not isinstance(func, ast.FunctionDef):
continue
for call in ast.walk(func):
if not (isinstance(call, ast.Call) and isinstance(call.func, ast.Name) and call.func.id == "_run_tests"):
continue
if len(call.args) >= 2:
path = _render_test_path(call.args[1])
prefix = "braintrust/integrations/"
if path and path.startswith(prefix):
name = path[len(prefix) :].split("/")[0]
if name in known:
mapping[name].add(func.name)
for keyword in call.keywords:
if keyword.arg == "env" and isinstance(keyword.value, ast.Dict):
for value in keyword.value.values:
if isinstance(value, ast.Constant) and value.value in known:
mapping[value.value].add(func.name)
for node in tree.body:
if isinstance(node, ast.FunctionDef):
for call in ast.walk(node):
if isinstance(call, ast.Call) and isinstance(call.func, ast.Name) and call.func.id == "_run_tests":
env = next((k.value for k in call.keywords if k.arg == "env"), None)
add(node.name, call.args[1:2], env)
elif (
isinstance(node, ast.Expr)
and isinstance(node.value, ast.Call)
and isinstance(node.value.func, ast.Name)
and node.value.func.id == "_register_matrix_sessions"
and node.value.args
and isinstance(node.value.args[0], ast.List)
):
for spec in node.value.args[0].elts:
if not isinstance(spec, ast.Dict):
continue
fields = {k.value: v for k, v in zip(spec.keys, spec.values) if isinstance(k, ast.Constant)}
name = fields.get("name")
if not (isinstance(name, ast.Constant) and isinstance(name.value, str)):
continue
tests = fields.get("tests")
test_paths = tests.elts if isinstance(tests, (ast.List, ast.Tuple)) else [tests] if tests else []
add(name.value, test_paths, fields.get("env"))
return dict(mapping)


Expand Down
Loading
Loading