diff --git a/.changesets/1792768307-aaecd177.yaml b/.changesets/1792768307-aaecd177.yaml new file mode 100644 index 000000000..52da12181 --- /dev/null +++ b/.changesets/1792768307-aaecd177.yaml @@ -0,0 +1,10 @@ +id: 1792768307-aaecd177 +features: + - core +targets: + - pythonv2 +type: feat +bump: patch +description: add preserveUnknownFields to keep undeclared fields in JSON request and response models +author: 2ynn +date: "2026-10-23" diff --git a/internal/features/tests.go b/internal/features/tests.go index a4031fcb5..c936b8b3f 100644 --- a/internal/features/tests.go +++ b/internal/features/tests.go @@ -781,6 +781,7 @@ const ( TestJsonlStreamTimeoutStillBoundsSlowStream TestJsonlStreamWithTimeoutStreamsToCompletion TestSharedEnumConstructorDefault + TestUnknownFieldsJsonRoundTrip ) func GetAllTests() []Test { diff --git a/internal/features/tests_generated.go b/internal/features/tests_generated.go index b6dd807de..01446360c 100644 --- a/internal/features/tests_generated.go +++ b/internal/features/tests_generated.go @@ -1502,6 +1502,8 @@ func (t Test) String() string { return "unions-weakly-typed-one-of-post-basic" case TestUnionsWeaklyTypedOneOfPostDeep: return "unions-weakly-typed-one-of-post-deep" + case TestUnknownFieldsJsonRoundTrip: + return "unknown-fields-json-round-trip" case TestWebhooksConsume: return "webhooks-consume" case TestWebhooksConsumeBadData: @@ -3022,6 +3024,8 @@ func TestFromString(s string) Test { return TestUnionsWeaklyTypedOneOfPostBasic case "unions-weakly-typed-one-of-post-deep": return TestUnionsWeaklyTypedOneOfPostDeep + case "unknown-fields-json-round-trip": + return TestUnknownFieldsJsonRoundTrip case "webhooks-consume": return TestWebhooksConsume case "webhooks-consume-bad-data": @@ -3792,6 +3796,7 @@ var testList = []Test{ TestUnionsUnionOfArrays, TestUnionsWeaklyTypedOneOfPostBasic, TestUnionsWeaklyTypedOneOfPostDeep, + TestUnknownFieldsJsonRoundTrip, TestWebhooksConsume, TestWebhooksConsumeBadData, TestWebhooksConsumeBadSignature, diff --git a/templates/perms.go b/templates/perms.go index 8c064d459..e4846b7f3 100644 --- a/templates/perms.go +++ b/templates/perms.go @@ -1543,6 +1543,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/auxiliary/tests/conftest.py.stmpl": 0o644, "templates/pythonv2/auxiliary/tests/helpers.py.stmpl": 0o644, "templates/pythonv2/auxiliary/tests/test_client.py.stmpl": 0o644, + "templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/__init__.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/_version.py.stmpl": 0o644, @@ -1552,7 +1553,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/auxiliary/{{getSourcePath}}/types": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/types/__init__.py": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/types/base64fileinput.py": 0o644, - "templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py": 0o644, + "templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils/__init__.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils/annotations.py": 0o644, @@ -1668,6 +1669,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/client-credentials/test_client_credentials_hook.py": 0o644, "templates/pythonv2/tests/common": 0o20000000755, "templates/pythonv2/tests/common/common_helpers.py": 0o644, + "templates/pythonv2/tests/common/unknown_fields_cases.py": 0o644, "templates/pythonv2/tests/custom-http": 0o20000000755, "templates/pythonv2/tests/custom-http/test_auth_additional.py": 0o644, "templates/pythonv2/tests/oauth2-password": 0o20000000755, @@ -1703,6 +1705,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/primary/test_telemetry.py.stmpl": 0o644, "templates/pythonv2/tests/primary/test_timeout_additional.py": 0o644, "templates/pythonv2/tests/primary/test_unions_additional.py": 0o644, + "templates/pythonv2/tests/primary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/primary/test_validation_errors.py": 0o644, "templates/pythonv2/tests/primary/test_x_ndjson_additional.py": 0o644, "templates/pythonv2/tests/quaternary": 0o20000000755, @@ -1712,6 +1715,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/quaternary/test_globals_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_jsonl_streaming_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_servers.py": 0o644, + "templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_validation_errors.py": 0o644, "templates/pythonv2/tests/secondary": 0o20000000755, "templates/pythonv2/tests/secondary/test_auth_additional.py": 0o644, @@ -1726,6 +1730,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/secondary/test_pagination.py": 0o644, "templates/pythonv2/tests/secondary/test_redirects_additional.py": 0o644, "templates/pythonv2/tests/secondary/test_servers.py": 0o644, + "templates/pythonv2/tests/secondary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/security-options": 0o20000000755, "templates/pythonv2/tests/security-options/test_auth_additional.py": 0o644, "templates/pythonv2/tests/tertiary": 0o20000000755, diff --git a/templates/templates/cli/features.ts b/templates/templates/cli/features.ts index a0c382e38..6cbf8cec5 100644 --- a/templates/templates/cli/features.ts +++ b/templates/templates/cli/features.ts @@ -359,6 +359,7 @@ function isTestSkipped(test: string): boolean { "parameters-path-parameter-formats", "shared-enum-constructor-default", "open-union-invalid-json-is-an-error", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/csharp/features.ts b/templates/templates/csharp/features.ts index 8c3bd71eb..6f8672df7 100644 --- a/templates/templates/csharp/features.ts +++ b/templates/templates/csharp/features.ts @@ -430,6 +430,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/go/features.ts b/templates/templates/go/features.ts index 1283baf4b..a82334d3c 100644 --- a/templates/templates/go/features.ts +++ b/templates/templates/go/features.ts @@ -427,6 +427,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/javav2/features.ts b/templates/templates/javav2/features.ts index 3e30be165..399f4fc36 100644 --- a/templates/templates/javav2/features.ts +++ b/templates/templates/javav2/features.ts @@ -478,6 +478,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/mcp-typescript/features.ts b/templates/templates/mcp-typescript/features.ts index fe3e88b41..d23ce8bfd 100644 --- a/templates/templates/mcp-typescript/features.ts +++ b/templates/templates/mcp-typescript/features.ts @@ -687,6 +687,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/php/features.ts b/templates/templates/php/features.ts index 0254cdade..3c421743e 100644 --- a/templates/templates/php/features.ts +++ b/templates/templates/php/features.ts @@ -572,6 +572,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl b/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl new file mode 100644 index 000000000..69da752f0 --- /dev/null +++ b/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl @@ -0,0 +1,32 @@ +{{- if and (sdkHasTests .Global.AST) .Global.Config.PreserveUnknownFields -}} +from typing import Any + +from pydantic import BaseModel + + +def drop_unexpected_fields(actual: Any, expected: Any) -> Any: + if isinstance(actual, BaseModel) and type(actual) is type(expected): + trimmed = actual.model_copy( + update={ + name: drop_unexpected_fields(getattr(actual, name), getattr(expected, name)) + for name in type(actual).model_fields + } + ) + if actual.__pydantic_extra__ is not None: + declared = "__pydantic_extra__" in type(actual).__annotations__ + expected_extra = expected.__pydantic_extra__ or {} + trimmed.__pydantic_extra__ = { + key: drop_unexpected_fields(value, expected_extra[key]) if key in expected_extra else value + for key, value in actual.__pydantic_extra__.items() + if declared or key in expected_extra + } + return trimmed + if isinstance(actual, list) and isinstance(expected, list) and len(actual) == len(expected): + return [drop_unexpected_fields(a, e) for a, e in zip(actual, expected)] + if isinstance(actual, dict) and isinstance(expected, dict): + return { + key: drop_unexpected_fields(value, expected[key]) if key in expected else value + for key, value in actual.items() + } + return actual +{{- end}} diff --git a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl similarity index 95% rename from templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py rename to templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl index 23751499a..5a7edd328 100644 --- a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py +++ b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl @@ -7,11 +7,14 @@ class BaseModel(PydanticBaseModel): model_config = ConfigDict( - populate_by_name=True, arbitrary_types_allowed=True, protected_namespaces=() + populate_by_name=True, arbitrary_types_allowed=True, protected_namespaces=(){{if .Global.Config.PreserveUnknownFields}}, extra="allow"{{end}} ) class Unset(BaseModel): +{{- if .Global.Config.PreserveUnknownFields}} + model_config = ConfigDict(extra="ignore") +{{end}} @model_serializer(mode="plain") def serialize_model(self): return UNSET_SENTINEL diff --git a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl index a3ea844b5..2062de6bb 100644 --- a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl +++ b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl @@ -278,7 +278,7 @@ def marshal_json(val, typ): m = marshaller(body=val) - d = m.model_dump(by_alias=True, mode="json", exclude_none=True) + d = m.model_dump(by_alias=True, mode="json", exclude_none={{if .Global.Config.PreserveUnknownFields}}getattr(m, "body") is None{{else}}True{{end}}) if len(d) == 0: return "" diff --git a/templates/templates/pythonv2/config.ts b/templates/templates/pythonv2/config.ts index 418cfa04a..886ef61cd 100644 --- a/templates/templates/pythonv2/config.ts +++ b/templates/templates/pythonv2/config.ts @@ -325,6 +325,15 @@ function getConfigFields( ValidationRegex: /^(true|false)$/.source, ValidationMessage: "true or false only", }, + preserveUnknownFields: { + Name: "preserveUnknownFields", + Required: false, + DefaultValue: false, + Description: + "Preserve undeclared fields in JSON request and response models, including nested models and schemas with additionalProperties: false. Explicitly typed additional properties retain their validation. Flattened operation arguments accept only declared fields, so undeclared top-level request fields require flattenRequests: false; nested models passed as arguments keep theirs. Disabled by default.", + ValidationRegex: /^(true|false)$/.source, + ValidationMessage: "true or false only", + }, responseSchemaValidation: { Name: "responseSchemaValidation", Required: false, diff --git a/templates/templates/pythonv2/includes/tests.ts b/templates/templates/pythonv2/includes/tests.ts index 7a5a2de87..9da99b1ac 100644 --- a/templates/templates/pythonv2/includes/tests.ts +++ b/templates/templates/pythonv2/includes/tests.ts @@ -151,22 +151,29 @@ function templateAssertion( operator = "is"; } - lines.push( - `assert ${assertionValue} ${operator} ${templateValue( - field, - example, - false, - { - usageContext: usageContext, - operation: usageContext?.Operation, - test: usageContext.Test, - isTest: true, - typedDictDisabled: true, - isResponse: true, - targetingMockServer: usageContext.UsingMockServer, - }, - )}`, - ); + const expected = templateValue(field, example, false, { + usageContext: usageContext, + operation: usageContext?.Operation, + test: usageContext.Test, + isTest: true, + typedDictDisabled: true, + isResponse: true, + targetingMockServer: usageContext.UsingMockServer, + }); + + if ( + context.Global.Config.PreserveUnknownFields && + ["class", "array", "map", "union"].includes(field.Type.Type.toString()) + ) { + addImport(".unknown_fields", "drop_unexpected_fields"); + lines.push(`expected = ${expected}`); + lines.push( + `assert drop_unexpected_fields(${assertionValue}, expected) == expected`, + ); + return lines; + } + + lines.push(`assert ${assertionValue} ${operator} ${expected}`); return lines; } diff --git a/templates/templates/pythonv2/model.py.stmpl b/templates/templates/pythonv2/model.py.stmpl index 4e13fc018..6603486da 100644 --- a/templates/templates/pythonv2/model.py.stmpl +++ b/templates/templates/pythonv2/model.py.stmpl @@ -7,6 +7,7 @@ {{- $hasOptionalField := gt (len $optionalFields) 0 -}} {{- $hasNullableField := gt (len $nullableFields) 0 -}} {{- $hasAdditionalProperties := hasAdditionalPropertiesField $type -}} +{{- $preserveUnknownFields := .Global.Config.PreserveUnknownFields -}} {{- if $hasAdditionalProperties}}{{addImport "pydantic" "ConfigDict"}}{{end -}} {{- $fields := (sortFields $type.Fields) -}} {{- $className := sanitizeClassName $type.Name }} @@ -93,11 +94,20 @@ class {{$className}}(BaseModel): for n, f in type(self).model_fields.items(): k = f.alias if f.alias is not None else n + {{- if $preserveUnknownFields}} + {{- /* Pydantic writes extras after declared fields, so an extra under a field's emitted key replaces that field's value */}} + {{- /* A nested model is serialized again from its own output, which may already use aliases */}} + key = k if info.by_alias or k in serialized else n + if key not in serialized and not info.exclude_none: + continue + val = serialized.pop(key, None) + {{- else}} if k not in serialized and n not in serialized and not info.exclude_none: continue val = serialized.get(k, serialized.get(n)) - {{- if $hasAdditionalProperties}} + {{- if $hasAdditionalProperties}} serialized.pop(k, serialized.pop(n, None)) + {{- end}} {{- end}} {{- if and $hasOptionalField $hasNullableField}} @@ -126,7 +136,7 @@ class {{$className}}(BaseModel): m[k] = val {{- end}} - {{- if $hasAdditionalProperties}} + {{- if or $hasAdditionalProperties $preserveUnknownFields}} for k, v in serialized.items(): m[k] = v {{- end}} diff --git a/templates/templates/pythonv2/tests/common/unknown_fields_cases.py b/templates/templates/pythonv2/tests/common/unknown_fields_cases.py new file mode 100644 index 000000000..2d5aac5d6 --- /dev/null +++ b/templates/templates/pythonv2/tests/common/unknown_fields_cases.py @@ -0,0 +1,28 @@ +UNKNOWN_FIELD_CASES = [ + ({"futureKind": "third", "futureCode": 3}, {"futureKind": "third", "futureCode": 3}), + ({"future": "value"}, {}), + ({"future": False}, {}), + ({"empty": {"future": True}}, {"empty": {}}), + ({"future": None}, {}), + ({"plainChoice": {"text": "value", "future": True}}, {"plainChoice": {"text": "value"}}), + ({"plainChoice": {"number": 3, "future": True}}, {"plainChoice": {"number": 3}}), + ({"requiredOnly": {"label": "required", "future": True}}, {"requiredOnly": {"label": "required"}}), + ({"future": 0}, {}), + ({"composed": {"displayName": "child", "category": "combined", "future": True}}, {"composed": {"displayName": "child", "category": "combined"}}), + ({"split": {"label": "split", "future": True}}, {"split": {"label": "split"}}), + ({"future": []}, {}), + ({"future": {"nested": [None, False, 0, "value"]}}, {}), + ({"child": {"displayName": "child", "future": True}}, {"child": {"displayName": "child"}}), + ({"child": {"displayName": "child", "display_name": "unknown"}}, {"child": {"displayName": "child"}}), + ({"child": {"displayName": "child", "child": {"displayName": "grandchild", "future": True}}}, {"child": {"displayName": "child", "child": {"displayName": "grandchild"}}}), + ({"children": [{"displayName": "child", "future": True}]}, {"children": [{"displayName": "child"}]}), + ({"lookup": {"one": {"displayName": "child", "future": True}}}, {"lookup": {"one": {"displayName": "child"}}}), + ({"choice": {"kind": "text", "text": "value", "future": True}}, {"choice": {"kind": "text", "text": "value"}}), + ({"choice": {"kind": "number", "number": 3, "future": True}}, {"choice": {"kind": "number", "number": 3}}), + ({"closed": {"label": "closed", "future": True}}, {"closed": {"label": "closed"}}), + ({"open": {"label": "open", "future": True}}, {"open": {"label": "open", "future": True}}), + ({"typed": {"label": "typed", "future": "value"}}, {"typed": {"label": "typed", "future": "value"}}), + ({"typedNodes": {"label": "nodes", "one": {"displayName": "child", "future": True}}}, {"typedNodes": {"label": "nodes", "one": {"displayName": "child"}}}), + ({"arbitrary": {"future": {"nullable": None}}}, {"arbitrary": {"future": {"nullable": None}}}), + ({"child": {"displayName": "child", "nullable": None}}, {"child": {"displayName": "child", "nullable": None}}), +] diff --git a/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py new file mode 100644 index 000000000..a87a602a5 --- /dev/null +++ b/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py @@ -0,0 +1,23 @@ +import json + +import pytest + +from openapi import SDK, utils +from openapi.models import shared +from .common_helpers import HTTPBIN_URL +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +@pytest.mark.parametrize("fields, known_fields", UNKNOWN_FIELD_CASES) +def test_unknown_fields_dropped_by_default(fields, known_fields): + model = shared.FieldEnvelopeInput.model_validate({"label": "root", **fields}) + expected = {"label": "root", **known_fields} + assert model.model_dump(mode="json", by_alias=True, exclude_none=True) == expected + assert json.loads(utils.marshal_json(model, shared.FieldEnvelopeInput)) == expected + + +def test_flattened_request_drops_nested_unknown_fields_by_default(): + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_flattened_unknown_fields(label="root", child={"displayName": "child", "future": True}) + assert response.http_meta.response.json()["json"] == {"label": "root", "child": {"displayName": "child"}} + assert response.object.model_dump(by_alias=True, exclude_none=True) == {"json": {"label": "root", "child": {"displayName": "child"}}} diff --git a/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py new file mode 100644 index 000000000..a5927e445 --- /dev/null +++ b/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py @@ -0,0 +1,60 @@ +import pytest + +from openapi import SDK, AsyncSDK +from openapi.models import shared +from .common_helpers import HTTPBIN_URL, record_test +from .unknown_fields import drop_unexpected_fields + + +@pytest.mark.parametrize("child", [ + {"displayName": "child", "future": True}, + {"displayName": "child", "display_name": "unknown"}, + {"displayName": "child", "child": {"displayName": "grandchild", "future": {"nested": [None, 0]}}}, + {"displayName": "child", "nullable": None, "future": None}, +]) +def test_flattened_request_keeps_nested_unknown_fields(child): + record_test("unknown-fields-json-round-trip") + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_unknown_fields(label="root", child=child) + payload = {"label": "root", "child": child} + assert response.raw_response.json()["json"] == payload + assert response.object.json_.model_dump(mode="json", by_alias=True) == payload + + +@pytest.mark.asyncio() +async def test_flattened_request_keeps_nested_unknown_fields_async(): + child = {"displayName": "child", "child": {"displayName": "grandchild", "future": {"nested": [None, 0]}}} + async with AsyncSDK(server_url=HTTPBIN_URL) as sdk: + response = await sdk.echo_unknown_fields(label="root", child=child) + payload = {"label": "root", "child": child} + assert response.raw_response.json()["json"] == payload + assert response.object.json_.model_dump(mode="json", by_alias=True) == payload + + +def test_flattened_request_keeps_unknown_fields_on_nested_models(): + child = shared.FieldNode(display_name="child", future=None) + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_flattened_unknown_fields(label="root", child=child) + assert response.raw_response.json()["json"] == {"label": "root", "child": {"displayName": "child", "future": None}} + + +def test_flattened_request_has_no_argument_for_top_level_unknown_fields(): + with SDK(server_url=HTTPBIN_URL) as sdk: + with pytest.raises(TypeError): + sdk.echo_flattened_unknown_fields(label="root", future=True) # type: ignore[call-arg] + + +def test_drop_unexpected_fields_ignores_only_undeclared_fields(): + expected = shared.TypedFields(label="typed") + assert drop_unexpected_fields(shared.TypedFields.model_validate({"label": "typed", "future": "value"}), expected) != expected + assert drop_unexpected_fields(shared.FieldNode.model_validate({"displayName": "child", "future": True}), shared.FieldNode(display_name="child")) == shared.FieldNode(display_name="child") + + +def test_drop_unexpected_fields_in_typed_additional_properties(): + expected = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "child"}}) + nested_unknown = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "child", "future": True}}) + extra_entry = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "child"}, "two": {"displayName": "other"}}) + changed_value = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "changed"}}) + assert drop_unexpected_fields(nested_unknown, expected) == expected + assert drop_unexpected_fields(extra_entry, expected) != expected + assert drop_unexpected_fields(changed_value, expected) != expected diff --git a/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py new file mode 100644 index 000000000..5b7c88ae8 --- /dev/null +++ b/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py @@ -0,0 +1,117 @@ +import json +from typing import Annotated, Any, List, Optional + +import httpx +import pydantic +import pytest + +from openapi import SDK, utils +from openapi.models import errors, shared +from openapi.types import UNSET, BaseModel, Nullable +from .common_helpers import HTTPBIN_URL, record_test +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +CASES = [fields for fields, _ in UNKNOWN_FIELD_CASES] + + +def echo(request): + sent = [] + with httpx.Client(event_hooks={"request": [sent.append]}) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + response = sdk.echo_unknown_fields(request=request) + return response, json.loads(sent[-1].content) + + +@pytest.mark.parametrize("fields", CASES) +@pytest.mark.parametrize("as_model", [False, True]) +def test_unknown_fields_echo(fields, as_model): + record_test("unknown-fields-json-round-trip") + payload = {"label": "root", **fields} + request = shared.FieldEnvelope.model_validate(payload) if as_model else payload + response, sent = echo(request) + assert sent == payload + assert response.json_.model_dump(mode="json", by_alias=True) == payload + assert json.loads(response.json_.model_dump_json(by_alias=True)) == payload + assert "headers" in response.model_dump() + + +def test_response_only_unknown_fields(): + payload = {"json": {"label": "root", "child": {"displayName": "child", "future": True}}, "future": {"value": 1}} + transport = httpx.MockTransport(lambda request: httpx.Response(200, json=payload)) + with httpx.Client(transport=transport) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + response = sdk.echo_unknown_fields(request={"label": "root"}) + dumped = response.model_dump(mode="json", by_alias=True) + assert "future" in dumped + assert "future" in dumped["json"]["child"] + + +def test_response_model_can_be_sent_again(): + payload = {"label": "root", "child": {"displayName": "child", "future": {"enabled": False}}} + first, _ = echo(payload) + second, sent = echo(first.json_.model_dump(by_alias=True)) + assert sent == payload + assert second.json_.model_dump(by_alias=True, exclude_none=True) == payload + + +def test_error_response_unknown_fields(): + payload = {"message": "invalid", "future": None, "detail": {"displayName": "child", "future": {"flag": True}}} + transport = httpx.MockTransport(lambda request: httpx.Response(400, json=payload)) + with httpx.Client(transport=transport) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + with pytest.raises(errors.EchoUnknownFieldsResponseBody) as caught: + sdk.echo_unknown_fields(request={"label": "root"}) + dumped = getattr(caught.value, "data").model_dump(mode="json", by_alias=True) + assert "future" in dumped + assert "future" in dumped["detail"] + + +def test_omitted_json_query_stays_empty(): + sent = [] + with httpx.Client(event_hooks={"request": [sent.append]}) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + sdk.echo_optional_json_query() + assert sent[-1].url.params["filter"] == "" + + +def test_typed_additional_properties_still_validate(): + with pytest.raises(pydantic.ValidationError): + shared.FieldEnvelope.model_validate({"label": "root", "typed": {"future": {"invalid": True}}}) + + +def test_unknown_fields_dump_filters(): + model = shared.FieldEnvelope.model_validate({"label": "root", "future": True, "future_null": None}) + assert "future_null" in model.model_dump() + assert "future_null" not in model.model_dump(exclude_none=True) + assert "future" not in model.model_dump(exclude={"future"}) + assert model.model_dump(include={"label"}) == {"label": "root"} + + +def test_base_model_unknown_field_policy(): + class CustomModel(BaseModel): + label: str + + model = CustomModel(label="value", future=None) + assert "future" in model.model_dump() + + +def test_invalid_nullable_value_becomes_unset_like_default_policy(): + node = shared.FieldNode.model_validate({"displayName": "child", "nullable": {"future": True}}) + assert node.nullable == UNSET + + +@pytest.mark.parametrize("typ", [Optional[int], Optional[List[int]], Optional[shared.FieldNode], Any]) +def test_top_level_none_serialization(typ): + assert utils.marshal_json(None, typ) == "" + + +def test_top_level_none_still_validates(): + with pytest.raises(pydantic.ValidationError): + utils.marshal_json(None, shared.FieldNode) + assert utils.marshal_json(None, Nullable[shared.FieldNode]) == "null" + + +def test_coerced_top_level_none_stays_empty(): + typ = Annotated[Optional[int], pydantic.BeforeValidator(lambda value: None)] + assert utils.marshal_json("empty", typ) == "" diff --git a/templates/templates/ruby/features.ts b/templates/templates/ruby/features.ts index 3435fe0de..fec3e49fc 100644 --- a/templates/templates/ruby/features.ts +++ b/templates/templates/ruby/features.ts @@ -475,6 +475,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/typescriptv2/features.ts b/templates/templates/typescriptv2/features.ts index becfb3d96..7a565f8eb 100644 --- a/templates/templates/typescriptv2/features.ts +++ b/templates/templates/typescriptv2/features.ts @@ -423,6 +423,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/unity/features.ts b/templates/templates/unity/features.ts index e69b828ac..4276f2af6 100644 --- a/templates/templates/unity/features.ts +++ b/templates/templates/unity/features.ts @@ -596,6 +596,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml b/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml index a61edf83b..98733ed2e 100644 --- a/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml @@ -35,6 +35,7 @@ python: responseFormat: envelope responseSchemaValidation: lenient flattenRequests: true + preserveUnknownFields: true pytestFilterWarnings: - error::RuntimeWarning # Ensure __del__ method warnings are raised diff --git a/tests/config/review/pythonv2/.speakeasy/gen.yaml b/tests/config/review/pythonv2/.speakeasy/gen.yaml index 7d4876daa..357bc40e1 100644 --- a/tests/config/review/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/review/pythonv2/.speakeasy/gen.yaml @@ -99,6 +99,7 @@ python: packageManager: uv packageName: openapi preApplyUnionDiscriminators: false + preserveUnknownFields: false pytestFilterWarnings: [] pytestTimeout: 0 rawResponseHelpers: false diff --git a/tests/config/secondary/pythonv2/.speakeasy/gen.yaml b/tests/config/secondary/pythonv2/.speakeasy/gen.yaml index 97ddd5bd2..fa854c67a 100644 --- a/tests/config/secondary/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/secondary/pythonv2/.speakeasy/gen.yaml @@ -38,6 +38,7 @@ python: webhooks: models/webhooks maxMethodParams: 5 flattenRequests: false + preserveUnknownFields: true packageName: openapi rawResponseHelpers: true responseFormat: flat diff --git a/tests/overlays/tertiary/typescriptv2/overlay.yaml b/tests/overlays/tertiary/typescriptv2/overlay.yaml new file mode 100644 index 000000000..fcc524d36 --- /dev/null +++ b/tests/overlays/tertiary/typescriptv2/overlay.yaml @@ -0,0 +1,9 @@ +overlay: 1.0.0 +info: + title: Overlay ./tests/specs/uber.yaml => tertiary typescriptv2 test spec + version: 0.0.0 +actions: + # Inline success and error bodies share a generated name, which collides on import + # without index modules, flat responses and the legacy naming scheme + - target: $.paths.*.*[?(@.operationId == 'echoUnknownFields')] + remove: true diff --git a/tests/specs/fragments/uber/unknown-fields.yaml b/tests/specs/fragments/uber/unknown-fields.yaml new file mode 100644 index 000000000..8ed2b359c --- /dev/null +++ b/tests/specs/fragments/uber/unknown-fields.yaml @@ -0,0 +1,256 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +paths: + /anything/unknown-fields-query: + get: + operationId: echoOptionalJsonQuery + description: Echoes an optional JSON query parameter without changing its omission semantics. + parameters: + - name: filter + in: query + required: false + content: + application/json: + schema: + title: UnknownFieldsQueryFilter + type: object + properties: + label: + type: string + responses: + '200': + description: The echoed request URL. + content: + application/json: + schema: + type: object + properties: + url: + type: string + /anything/unknown-fields: + post: + operationId: echoUnknownFields + x-speakeasy-exclude-test-sdk: [javav2] + description: Echoes nested objects to exercise preservation of undeclared fields in requests and responses. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FieldEnvelope' + responses: + '200': + description: The echoed JSON request and transport metadata. + content: + application/json: + schema: + type: object + required: [json] + properties: + json: + $ref: '#/components/schemas/FieldEnvelope' + '400': + description: A structured error with additional server fields. + content: + application/json: + schema: + type: object + required: [message] + properties: + message: + type: string + detail: + $ref: '#/components/schemas/FieldNode' + /anything/unknown-fields-flattened: + post: + operationId: echoFlattenedUnknownFields + description: Echoes a body small enough to be flattened into operation arguments, so undeclared fields can only travel in nested objects. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FlattenedFields' + responses: + '200': + description: The echoed JSON request. + content: + application/json: + schema: + type: object + required: [json] + properties: + json: + $ref: '#/components/schemas/FlattenedFields' +components: + schemas: + FieldEnvelope: + type: object + required: [label] + properties: + label: + type: string + child: + $ref: '#/components/schemas/FieldNode' + children: + type: array + items: + $ref: '#/components/schemas/FieldNode' + lookup: + type: object + additionalProperties: + $ref: '#/components/schemas/FieldNode' + choice: + oneOf: + - $ref: '#/components/schemas/TextChoice' + - $ref: '#/components/schemas/NumberChoice' + discriminator: + propertyName: kind + mapping: + text: '#/components/schemas/TextChoice' + number: '#/components/schemas/NumberChoice' + plainChoice: + anyOf: + - $ref: '#/components/schemas/TextFields' + - $ref: '#/components/schemas/NumberFields' + requiredOnly: + $ref: '#/components/schemas/RequiredFields' + composed: + allOf: + - $ref: '#/components/schemas/FieldNode' + - type: object + properties: + category: + type: string + split: + $ref: '#/components/schemas/SplitFields' + empty: + $ref: '#/components/schemas/EmptyFields' + open: + $ref: '#/components/schemas/OpenFields' + typed: + $ref: '#/components/schemas/TypedFields' + typedNodes: + $ref: '#/components/schemas/TypedNodes' + closed: + $ref: '#/components/schemas/ClosedFields' + arbitrary: {} + futureKind: + type: string + enum: [first, second] + x-speakeasy-unknown-values: allow + futureCode: + type: integer + enum: [1, 2] + x-speakeasy-unknown-values: allow + FieldNode: + type: object + required: [displayName] + properties: + displayName: + type: string + nullable: + type: [string, 'null'] + child: + $ref: '#/components/schemas/FieldNode' + TextChoice: + type: object + required: [kind, text] + properties: + kind: + $ref: '#/components/schemas/TextChoiceKind' + text: + type: string + NumberChoice: + type: object + required: [kind, number] + properties: + kind: + $ref: '#/components/schemas/NumberChoiceKind' + number: + type: integer + TextChoiceKind: + type: string + enum: [text] + NumberChoiceKind: + type: string + enum: [number] + OpenFields: + type: object + additionalProperties: true + properties: + label: + type: string + TypedFields: + type: object + additionalProperties: + type: string + properties: + label: + type: string + TypedNodes: + type: object + description: Additional properties typed as an object schema, whose values can carry undeclared fields of their own. + additionalProperties: + $ref: '#/components/schemas/FieldNode' + properties: + label: + type: string + ClosedFields: + type: object + additionalProperties: false + properties: + label: + type: string + TextFields: + type: object + required: [text] + properties: + text: + type: string + label: + type: string + NumberFields: + type: object + required: [number] + properties: + number: + type: integer + label: + type: string + RequiredFields: + type: object + required: [label] + properties: + label: + type: string + SplitFields: + type: object + properties: + label: + type: string + id: + type: string + readOnly: true + secret: + type: string + writeOnly: true + EmptyFields: + type: object + additionalProperties: false + FlattenedFields: + type: object + description: A body with few enough properties to be flattened into operation arguments. + required: [label] + properties: + label: + type: string + child: + $ref: '#/components/schemas/FieldNode' diff --git a/zSDKs/sdk-pythonv2/.speakeasy/gen.lock b/zSDKs/sdk-pythonv2/.speakeasy/gen.lock index e5217c27c..4dd87f1bf 100644 --- a/zSDKs/sdk-pythonv2/.speakeasy/gen.lock +++ b/zSDKs/sdk-pythonv2/.speakeasy/gen.lock @@ -6,7 +6,7 @@ management: speakeasyVersion: internal generationVersion: internal releaseVersion: 0.0.1 - configChecksum: d67daf2ec44e985625b98d32573578c6 + configChecksum: 908ffac3f2935a4cd93222913042d257 repoURL: https://github.com/speakeasy-sdks/test-sdk installationURL: https://github.com/speakeasy-sdks/test-sdk.git features: @@ -746,7 +746,7 @@ trackedFiles: src/speakeasy/new_openapi/types/base64fileinput.py: last_write_checksum: sha1:1522687ae3398374c35710cad993a6e82b5ab99d src/speakeasy/new_openapi/types/basemodel.py: - last_write_checksum: sha1:6295cc5750e00e0323902092f7e52549937b0b9c + last_write_checksum: sha1:16e62753ef3b7f377dd826eaff379a519e1096d3 src/speakeasy/new_openapi/types_.py: last_write_checksum: sha1:05e36f6428f04b1ea7d2609aa55f3966868b58f0 src/speakeasy/new_openapi/utils/__init__.py: diff --git a/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml b/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml index 7d4876daa..357bc40e1 100644 --- a/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml +++ b/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml @@ -99,6 +99,7 @@ python: packageManager: uv packageName: openapi preApplyUnionDiscriminators: false + preserveUnknownFields: false pytestFilterWarnings: [] pytestTimeout: 0 rawResponseHelpers: false diff --git a/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py b/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py index 06c13bfcd..5c4819c8b 100644 --- a/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py +++ b/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py @@ -37,7 +37,9 @@ def __bool__(self) -> Literal[False]: ) -def _unrecognized_serialization(cls: Type[Union[str, int]]) -> core_schema.PlainSerializerFunctionSerSchema: +def _unrecognized_serialization( + cls: Type[Union[str, int]], +) -> core_schema.PlainSerializerFunctionSerSchema: def serialize(value: Any, info: core_schema.SerializationInfo) -> Any: # Decline other values so union serialization tries the remaining members if not isinstance(value, cls): @@ -51,10 +53,12 @@ def serialize(value: Any, info: core_schema.SerializationInfo) -> Any: class UnrecognizedStr(str): @classmethod - def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> core_schema.CoreSchema: + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: Any + ) -> core_schema.CoreSchema: # Make UnrecognizedStr only work in lax mode, not strict mode # This makes it a "fallback" option when more specific types (like Literals) don't match - def validate_lax(v: Any) -> 'UnrecognizedStr': + def validate_lax(v: Any) -> "UnrecognizedStr": if isinstance(v, cls): return v return cls(str(v)) @@ -63,10 +67,12 @@ def validate_lax(v: Any) -> 'UnrecognizedStr': # This forces Pydantic to prefer other union members in strict mode # and only fall back to UnrecognizedStr in lax mode return core_schema.lax_or_strict_schema( - lax_schema=core_schema.chain_schema([ - core_schema.str_schema(), - core_schema.no_info_plain_validator_function(validate_lax) - ]), + lax_schema=core_schema.chain_schema( + [ + core_schema.str_schema(), + core_schema.no_info_plain_validator_function(validate_lax), + ] + ), strict_schema=core_schema.none_schema(), # Always fails in strict mode serialization=_unrecognized_serialization(cls), ) @@ -74,18 +80,23 @@ def validate_lax(v: Any) -> 'UnrecognizedStr': class UnrecognizedInt(int): @classmethod - def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> core_schema.CoreSchema: + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: Any + ) -> core_schema.CoreSchema: # Make UnrecognizedInt only work in lax mode, not strict mode # This makes it a "fallback" option when more specific types (like Literals) don't match - def validate_lax(v: Any) -> 'UnrecognizedInt': + def validate_lax(v: Any) -> "UnrecognizedInt": if isinstance(v, cls): return v return cls(int(v)) + return core_schema.lax_or_strict_schema( - lax_schema=core_schema.chain_schema([ - core_schema.int_schema(), - core_schema.no_info_plain_validator_function(validate_lax) - ]), + lax_schema=core_schema.chain_schema( + [ + core_schema.int_schema(), + core_schema.no_info_plain_validator_function(validate_lax), + ] + ), strict_schema=core_schema.none_schema(), # Always fails in strict mode serialization=_unrecognized_serialization(cls), )