Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions integration_tests/_contract_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
from copy import deepcopy
from typing import Any

import typing_extensions

from integration_tests import _contract_surface as surface, _contract_validation as validation


Expand Down Expand Up @@ -235,6 +237,28 @@ def build_released_api_contract(
released_exports = set(released_export_order)
current_export_names = set(current_exports)
if release_policy is not None:
promoted_top_level_typed_dicts = {
entry["class_name"]: set(entry["names"])
for entry in release_policy.public_typed_dicts
if entry["module"] == "agents"
}
for name in sorted(current_export_names - released_exports):
value = getattr(agents, name)
if not typing_extensions.is_typeddict(value):
continue
if name not in promoted_top_level_typed_dicts:
raise ValueError(
f"Cannot promote new top-level TypedDict agents.{name} without a "
"public_typed_dicts policy entry for module 'agents'"
)
missing_fields = sorted(
set(value.__annotations__) - promoted_top_level_typed_dicts[name]
)
if missing_fields:
raise ValueError(
f"Cannot promote new top-level TypedDict agents.{name} without "
f"public_typed_dicts policy fields: {missing_fields!r}"
)
promoted_top_level_type_aliases = {
entry["name"]
for entry in release_policy.public_type_aliases
Expand Down
2 changes: 1 addition & 1 deletion tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ Compare test counts, skips, warnings, assertions, and lifecycle coverage as well

Release compatibility unit tests must exercise policy and validation logic with explicit constructed modules instead of inspecting the current checkout's shared import state. The prospective release-contract job validates the current source checkout once in a dedicated Python process, and the packaged integration profiles validate real wheel, sdist, optional-extra, and platform surfaces in isolated environments. Keep the combined serial focused runtime of release compatibility unit tests below 1.5 seconds and each case below 100 milliseconds in normal conditions. Tests that build or install distributions, isolate imports in new interpreters, access the network, start containers, or require external services belong in `integration_tests/` instead. Run the concrete local Docker-backed contracts with `make integration-tests-containers`; ordinary `make tests` must not pull container images. The released API manifest permits compatible suffix additions and records enum member construction from `Enum.__new__` so CPython's version-dependent enum metaclass signature is not treated as an SDK change. Historical `RunState` fixtures must be produced by the recorded historical writer rather than by editing a current payload's schema version; the corpus README documents the explicit canonical-reader exception for schema versions that never had a corresponding writer.

The released API manifest is a rolling latest-release contract. After a release PR has updated `pyproject.toml`, check out the clean release branch locally and run `make update-released-api-contract VERSION=<version>` before requesting final review. The command first rejects any incompatibility with the committed contract, then freezes the candidate's current top-level exports, every inspectable top-level class or function signature and execution kind, and every newly exported SDK-owned inspectable class or function in a tracked public submodule. Qualified submodule callables remain tracked in later releases. Callable contracts include Pydantic model field names and defaults plus the binding, signature, and execution kind of each public callable member declared directly on an exported class or inherited from an SDK-owned base class. Exact Python functions are classified as synchronous functions, generators, coroutines, or asynchronous generators, and decorated functions use their caller-visible standard `inspect.signature()` contract. Methods declared only by third-party base classes and arbitrary descriptors remain outside the contract. Before regeneration, add newly documented class properties or factory-result properties to `public_properties`, selected public `TypedDict` fields to `public_typed_dicts`, newly intended cross-module import identities to `canonical_imports`, and optional module/export declarations to `modules` in `tests/fixtures/released_api_contract_policy.json`; those policy decisions are deliberately not inferred from implementation modules. The generator records each selected `TypedDict` field's requiredness and declared annotation without enrolling arbitrary `TypedDict` classes or fields. It merges the curated policy into the prospective and released contracts and freezes declared unsupported platforms so validation remains portable. The v0.19.4 baseline includes base-package submodule paths, canonical identities, and documented result properties used by its shipped docs and examples; optional-extra and experimental paths remain outside this base contract. After rebasing the release branch, run `make check-released-api-contract VERSION=<version>` and regenerate only when it reports drift. No GitHub workflow imports candidate code with write credentials for this update.
The released API manifest is a rolling latest-release contract. After a release PR has updated `pyproject.toml`, check out the clean release branch locally and run `make update-released-api-contract VERSION=<version>` before requesting final review. The command first rejects any incompatibility with the committed contract, then freezes the candidate's current top-level exports, every inspectable top-level class or function signature and execution kind, and every newly exported SDK-owned inspectable class or function in a tracked public submodule. Qualified submodule callables remain tracked in later releases. Callable contracts include Pydantic model field names and defaults plus the binding, signature, and execution kind of each public callable member declared directly on an exported class or inherited from an SDK-owned base class. Exact Python functions are classified as synchronous functions, generators, coroutines, or asynchronous generators, and decorated functions use their caller-visible standard `inspect.signature()` contract. Methods declared only by third-party base classes and arbitrary descriptors remain outside the contract. Before regeneration, add newly documented class properties or factory-result properties to `public_properties`, selected public `TypedDict` fields to `public_typed_dicts`, newly intended cross-module import identities to `canonical_imports`, and optional module/export declarations to `modules` in `tests/fixtures/released_api_contract_policy.json`; those policy decisions are deliberately not inferred from implementation modules. The generator records each selected `TypedDict` field's requiredness and declared annotation without enrolling arbitrary `TypedDict` classes or fields. For a newly exported top-level `TypedDict`, add a `public_typed_dicts` entry for module `agents` that lists every field, including inherited fields. The existing `prospective-release-contract` CI job rejects missing entries or omitted fields on the implementation PR, before release preparation. Previously released exports retain their curated coverage; this check does not recursively discover nested types. It merges the curated policy into the prospective and released contracts and freezes declared unsupported platforms so validation remains portable. The v0.19.4 baseline includes base-package submodule paths, canonical identities, and documented result properties used by its shipped docs and examples; optional-extra and experimental paths remain outside this base contract. After rebasing the release branch, run `make check-released-api-contract VERSION=<version>` and regenerate only when it reports drift. No GitHub workflow imports candidate code with write credentials for this update.

## Snapshots

Expand Down
8 changes: 8 additions & 0 deletions tests/fixtures/released_api_contract_policy.json
Original file line number Diff line number Diff line change
Expand Up @@ -792,6 +792,14 @@
}
],
"public_typed_dicts": [
{
"class_name": "WebSearchToolImageSettings",
"module": "agents",
"names": [
"max_results",
"caption"
]
},
{
"class_name": "ModelStepSpec",
"module": "agents.testing.model",
Expand Down
118 changes: 118 additions & 0 deletions tests/test_released_api_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -1180,6 +1180,119 @@ def test_recursive_type_alias_type_is_rejected() -> None:
)


@pytest.mark.parametrize(
"policy_names", [None, ["max_results"], ["caption"]], ids=["missing", "no-caption", "no-limit"]
)
@pytest.mark.parametrize("already_released", [False, True], ids=["new-export", "released-export"])
def test_new_top_level_typed_dict_requires_complete_policy(
policy_names: list[str] | None, already_released: bool
) -> None:
from agents import WebSearchToolImageSettings

agents_module = SimpleNamespace(
__all__=["WebSearchToolImageSettings"],
WebSearchToolImageSettings=WebSearchToolImageSettings,
)
contract: dict[str, Any] = {
"baseline": "v0.22.0",
"baseline_commit": "a" * 40,
"required_top_level_exports": ["WebSearchToolImageSettings"] if already_released else [],
"canonical_imports": [],
"public_modules": ["agents"],
"callables": {},
}
policy_entries = (
()
if policy_names is None
else (
{
"module": "agents",
"class_name": "WebSearchToolImageSettings",
"names": policy_names,
},
)
)

def promote() -> dict[str, Any]:
return build_released_api_contract(
contract,
baseline="v0.22.1",
baseline_commit="b" * 40,
agents_module=agents_module,
release_policy=_release_policy({}, public_typed_dicts=policy_entries),
)

if already_released:
updated = promote()
assert updated["required_top_level_exports"] == ["WebSearchToolImageSettings"]
else:
with pytest.raises(ValueError, match="WebSearchToolImageSettings.*public_typed_dicts"):
promote()


@pytest.mark.parametrize("drift", ["removed", "required", "type"])
def test_web_search_image_settings_policy_freezes_fields(drift: str) -> None:
from agents import WebSearchToolImageSettings

agents_module = SimpleNamespace(
__all__=["WebSearchToolImageSettings"],
WebSearchToolImageSettings=WebSearchToolImageSettings,
)
policy = load_submodule_export_policy(CONTRACT.with_name("released_api_contract_policy.json"))
image_settings_policy = tuple(
entry
for entry in policy.public_typed_dicts
if entry["module"] == "agents" and entry["class_name"] == "WebSearchToolImageSettings"
)
contract: dict[str, Any] = {
"baseline": "v0.22.0",
"baseline_commit": "a" * 40,
"required_top_level_exports": [],
"canonical_imports": [],
"public_modules": ["agents"],
"callables": {},
}
updated = build_released_api_contract(
contract,
baseline="v0.22.1",
baseline_commit="b" * 40,
agents_module=agents_module,
release_policy=_release_policy({}, public_typed_dicts=image_settings_policy),
)

assert updated["public_typed_dicts"] == [
{
"module": "agents",
"class_name": "WebSearchToolImageSettings",
"fields": [
{"name": "max_results", "required": False, "annotation": "int"},
{"name": "caption", "required": False, "annotation": "bool"},
],
}
]
assert validate_released_api_contract(updated, agents_module=agents_module) == []

class RemovedField(TypedDict, total=False):
max_results: int

class RequiredField(TypedDict):
max_results: int
caption: bool

class ChangedType(TypedDict, total=False):
max_results: str
caption: bool

agents_module.WebSearchToolImageSettings = {
"removed": RemovedField,
"required": RequiredField,
"type": ChangedType,
}[drift]
errors = validate_released_api_contract(updated, agents_module=agents_module)
assert errors
assert all("changed its released TypedDict field contract" in error for error in errors)


def test_curated_public_typed_dict_contract_detects_field_shape_drift(
monkeypatch: pytest.MonkeyPatch,
) -> None:
Expand Down Expand Up @@ -3860,6 +3973,11 @@ def test_repository_release_policy_declares_public_state_surfaces() -> None:
},
}
assert policy.public_typed_dicts == (
{
"class_name": "WebSearchToolImageSettings",
"module": "agents",
"names": ["max_results", "caption"],
},
{
"class_name": "ModelStepSpec",
"module": "agents.testing.model",
Expand Down