From 876df5f7d63fee8c087a2f79d1bd70dd808c4458 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Fri, 23 Oct 2026 13:06:06 -0400 Subject: [PATCH] feat(python): add preserveUnknownFields option Pydantic drops fields a model does not declare, so undeclared fields in nested JSON request and response bodies were lost on validation and on serialization back to the API. Add python.preserveUnknownFields (default false). When enabled: - generated models and error bodies allow extra fields, including schemas with additionalProperties: false; explicitly typed additional properties still validate - model serializers read each declared field from the key Pydantic emitted (alias or field name, including the alias-keyed output a nested model is serialized from again) and copy the remaining keys back out - marshal_json keeps nested nulls and only excludes a top-level None - UNSET ignores extra fields - generated test assertions compare only the fields their examples declare, since mock responses can carry more; declared additional properties keep their exact key set and are compared recursively basemodel.py becomes a template, so the review SDK copy now passes through the Python formatter. The option is covered by an echo fragment exercised in the primary (disabled), secondary and quaternary variants. --- .changesets/1792768307-aaecd177.yaml | 10 + internal/features/tests.go | 1 + internal/features/tests_generated.go | 5 + templates/perms.go | 7 +- templates/templates/cli/features.ts | 1 + templates/templates/csharp/features.ts | 1 + templates/templates/go/features.ts | 1 + templates/templates/javav2/features.ts | 1 + .../templates/mcp-typescript/features.ts | 1 + templates/templates/php/features.ts | 1 + .../auxiliary/tests/unknown_fields.py.stmpl | 32 +++ .../{basemodel.py => basemodel.py.stmpl} | 5 +- .../utils/serializers.py.stmpl | 2 +- templates/templates/pythonv2/config.ts | 9 + .../templates/pythonv2/includes/tests.ts | 39 +-- templates/templates/pythonv2/model.py.stmpl | 14 +- .../tests/common/unknown_fields_cases.py | 28 ++ .../primary/test_unknown_fields_additional.py | 23 ++ .../test_unknown_fields_additional.py | 60 ++++ .../test_unknown_fields_additional.py | 117 ++++++++ templates/templates/ruby/features.ts | 1 + templates/templates/typescriptv2/features.ts | 1 + templates/templates/unity/features.ts | 1 + .../quaternary/pythonv2/.speakeasy/gen.yaml | 1 + .../review/pythonv2/.speakeasy/gen.yaml | 1 + .../secondary/pythonv2/.speakeasy/gen.yaml | 1 + .../tertiary/typescriptv2/overlay.yaml | 9 + .../specs/fragments/uber/unknown-fields.yaml | 256 ++++++++++++++++++ zSDKs/sdk-pythonv2/.speakeasy/gen.lock | 4 +- zSDKs/sdk-pythonv2/.speakeasy/gen.yaml | 1 + .../speakeasy/new_openapi/types/basemodel.py | 37 ++- 31 files changed, 635 insertions(+), 36 deletions(-) create mode 100644 .changesets/1792768307-aaecd177.yaml create mode 100644 templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl rename templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/{basemodel.py => basemodel.py.stmpl} (95%) create mode 100644 templates/templates/pythonv2/tests/common/unknown_fields_cases.py create mode 100644 templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py create mode 100644 templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py create mode 100644 templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py create mode 100644 tests/overlays/tertiary/typescriptv2/overlay.yaml create mode 100644 tests/specs/fragments/uber/unknown-fields.yaml diff --git a/.changesets/1792768307-aaecd177.yaml b/.changesets/1792768307-aaecd177.yaml new file mode 100644 index 000000000..52da12181 --- /dev/null +++ b/.changesets/1792768307-aaecd177.yaml @@ -0,0 +1,10 @@ +id: 1792768307-aaecd177 +features: + - core +targets: + - pythonv2 +type: feat +bump: patch +description: add preserveUnknownFields to keep undeclared fields in JSON request and response models +author: 2ynn +date: "2026-10-23" diff --git a/internal/features/tests.go b/internal/features/tests.go index a4031fcb5..c936b8b3f 100644 --- a/internal/features/tests.go +++ b/internal/features/tests.go @@ -781,6 +781,7 @@ const ( TestJsonlStreamTimeoutStillBoundsSlowStream TestJsonlStreamWithTimeoutStreamsToCompletion TestSharedEnumConstructorDefault + TestUnknownFieldsJsonRoundTrip ) func GetAllTests() []Test { diff --git a/internal/features/tests_generated.go b/internal/features/tests_generated.go index b6dd807de..01446360c 100644 --- a/internal/features/tests_generated.go +++ b/internal/features/tests_generated.go @@ -1502,6 +1502,8 @@ func (t Test) String() string { return "unions-weakly-typed-one-of-post-basic" case TestUnionsWeaklyTypedOneOfPostDeep: return "unions-weakly-typed-one-of-post-deep" + case TestUnknownFieldsJsonRoundTrip: + return "unknown-fields-json-round-trip" case TestWebhooksConsume: return "webhooks-consume" case TestWebhooksConsumeBadData: @@ -3022,6 +3024,8 @@ func TestFromString(s string) Test { return TestUnionsWeaklyTypedOneOfPostBasic case "unions-weakly-typed-one-of-post-deep": return TestUnionsWeaklyTypedOneOfPostDeep + case "unknown-fields-json-round-trip": + return TestUnknownFieldsJsonRoundTrip case "webhooks-consume": return TestWebhooksConsume case "webhooks-consume-bad-data": @@ -3792,6 +3796,7 @@ var testList = []Test{ TestUnionsUnionOfArrays, TestUnionsWeaklyTypedOneOfPostBasic, TestUnionsWeaklyTypedOneOfPostDeep, + TestUnknownFieldsJsonRoundTrip, TestWebhooksConsume, TestWebhooksConsumeBadData, TestWebhooksConsumeBadSignature, diff --git a/templates/perms.go b/templates/perms.go index 8c064d459..e4846b7f3 100644 --- a/templates/perms.go +++ b/templates/perms.go @@ -1543,6 +1543,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/auxiliary/tests/conftest.py.stmpl": 0o644, "templates/pythonv2/auxiliary/tests/helpers.py.stmpl": 0o644, "templates/pythonv2/auxiliary/tests/test_client.py.stmpl": 0o644, + "templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/__init__.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/_version.py.stmpl": 0o644, @@ -1552,7 +1553,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/auxiliary/{{getSourcePath}}/types": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/types/__init__.py": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/types/base64fileinput.py": 0o644, - "templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py": 0o644, + "templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils/__init__.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils/annotations.py": 0o644, @@ -1668,6 +1669,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/client-credentials/test_client_credentials_hook.py": 0o644, "templates/pythonv2/tests/common": 0o20000000755, "templates/pythonv2/tests/common/common_helpers.py": 0o644, + "templates/pythonv2/tests/common/unknown_fields_cases.py": 0o644, "templates/pythonv2/tests/custom-http": 0o20000000755, "templates/pythonv2/tests/custom-http/test_auth_additional.py": 0o644, "templates/pythonv2/tests/oauth2-password": 0o20000000755, @@ -1703,6 +1705,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/primary/test_telemetry.py.stmpl": 0o644, "templates/pythonv2/tests/primary/test_timeout_additional.py": 0o644, "templates/pythonv2/tests/primary/test_unions_additional.py": 0o644, + "templates/pythonv2/tests/primary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/primary/test_validation_errors.py": 0o644, "templates/pythonv2/tests/primary/test_x_ndjson_additional.py": 0o644, "templates/pythonv2/tests/quaternary": 0o20000000755, @@ -1712,6 +1715,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/quaternary/test_globals_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_jsonl_streaming_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_servers.py": 0o644, + "templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_validation_errors.py": 0o644, "templates/pythonv2/tests/secondary": 0o20000000755, "templates/pythonv2/tests/secondary/test_auth_additional.py": 0o644, @@ -1726,6 +1730,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/secondary/test_pagination.py": 0o644, "templates/pythonv2/tests/secondary/test_redirects_additional.py": 0o644, "templates/pythonv2/tests/secondary/test_servers.py": 0o644, + "templates/pythonv2/tests/secondary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/security-options": 0o20000000755, "templates/pythonv2/tests/security-options/test_auth_additional.py": 0o644, "templates/pythonv2/tests/tertiary": 0o20000000755, diff --git a/templates/templates/cli/features.ts b/templates/templates/cli/features.ts index a0c382e38..6cbf8cec5 100644 --- a/templates/templates/cli/features.ts +++ b/templates/templates/cli/features.ts @@ -359,6 +359,7 @@ function isTestSkipped(test: string): boolean { "parameters-path-parameter-formats", "shared-enum-constructor-default", "open-union-invalid-json-is-an-error", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/csharp/features.ts b/templates/templates/csharp/features.ts index 8c3bd71eb..6f8672df7 100644 --- a/templates/templates/csharp/features.ts +++ b/templates/templates/csharp/features.ts @@ -430,6 +430,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/go/features.ts b/templates/templates/go/features.ts index 1283baf4b..a82334d3c 100644 --- a/templates/templates/go/features.ts +++ b/templates/templates/go/features.ts @@ -427,6 +427,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/javav2/features.ts b/templates/templates/javav2/features.ts index 3e30be165..399f4fc36 100644 --- a/templates/templates/javav2/features.ts +++ b/templates/templates/javav2/features.ts @@ -478,6 +478,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/mcp-typescript/features.ts b/templates/templates/mcp-typescript/features.ts index fe3e88b41..d23ce8bfd 100644 --- a/templates/templates/mcp-typescript/features.ts +++ b/templates/templates/mcp-typescript/features.ts @@ -687,6 +687,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/php/features.ts b/templates/templates/php/features.ts index 0254cdade..3c421743e 100644 --- a/templates/templates/php/features.ts +++ b/templates/templates/php/features.ts @@ -572,6 +572,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl b/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl new file mode 100644 index 000000000..69da752f0 --- /dev/null +++ b/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl @@ -0,0 +1,32 @@ +{{- if and (sdkHasTests .Global.AST) .Global.Config.PreserveUnknownFields -}} +from typing import Any + +from pydantic import BaseModel + + +def drop_unexpected_fields(actual: Any, expected: Any) -> Any: + if isinstance(actual, BaseModel) and type(actual) is type(expected): + trimmed = actual.model_copy( + update={ + name: drop_unexpected_fields(getattr(actual, name), getattr(expected, name)) + for name in type(actual).model_fields + } + ) + if actual.__pydantic_extra__ is not None: + declared = "__pydantic_extra__" in type(actual).__annotations__ + expected_extra = expected.__pydantic_extra__ or {} + trimmed.__pydantic_extra__ = { + key: drop_unexpected_fields(value, expected_extra[key]) if key in expected_extra else value + for key, value in actual.__pydantic_extra__.items() + if declared or key in expected_extra + } + return trimmed + if isinstance(actual, list) and isinstance(expected, list) and len(actual) == len(expected): + return [drop_unexpected_fields(a, e) for a, e in zip(actual, expected)] + if isinstance(actual, dict) and isinstance(expected, dict): + return { + key: drop_unexpected_fields(value, expected[key]) if key in expected else value + for key, value in actual.items() + } + return actual +{{- end}} diff --git a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl similarity index 95% rename from templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py rename to templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl index 23751499a..5a7edd328 100644 --- a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py +++ b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl @@ -7,11 +7,14 @@ class BaseModel(PydanticBaseModel): model_config = ConfigDict( - populate_by_name=True, arbitrary_types_allowed=True, protected_namespaces=() + populate_by_name=True, arbitrary_types_allowed=True, protected_namespaces=(){{if .Global.Config.PreserveUnknownFields}}, extra="allow"{{end}} ) class Unset(BaseModel): +{{- if .Global.Config.PreserveUnknownFields}} + model_config = ConfigDict(extra="ignore") +{{end}} @model_serializer(mode="plain") def serialize_model(self): return UNSET_SENTINEL diff --git a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl index a3ea844b5..2062de6bb 100644 --- a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl +++ b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl @@ -278,7 +278,7 @@ def marshal_json(val, typ): m = marshaller(body=val) - d = m.model_dump(by_alias=True, mode="json", exclude_none=True) + d = m.model_dump(by_alias=True, mode="json", exclude_none={{if .Global.Config.PreserveUnknownFields}}getattr(m, "body") is None{{else}}True{{end}}) if len(d) == 0: return "" diff --git a/templates/templates/pythonv2/config.ts b/templates/templates/pythonv2/config.ts index 418cfa04a..886ef61cd 100644 --- a/templates/templates/pythonv2/config.ts +++ b/templates/templates/pythonv2/config.ts @@ -325,6 +325,15 @@ function getConfigFields( ValidationRegex: /^(true|false)$/.source, ValidationMessage: "true or false only", }, + preserveUnknownFields: { + Name: "preserveUnknownFields", + Required: false, + DefaultValue: false, + Description: + "Preserve undeclared fields in JSON request and response models, including nested models and schemas with additionalProperties: false. Explicitly typed additional properties retain their validation. Flattened operation arguments accept only declared fields, so undeclared top-level request fields require flattenRequests: false; nested models passed as arguments keep theirs. Disabled by default.", + ValidationRegex: /^(true|false)$/.source, + ValidationMessage: "true or false only", + }, responseSchemaValidation: { Name: "responseSchemaValidation", Required: false, diff --git a/templates/templates/pythonv2/includes/tests.ts b/templates/templates/pythonv2/includes/tests.ts index 7a5a2de87..9da99b1ac 100644 --- a/templates/templates/pythonv2/includes/tests.ts +++ b/templates/templates/pythonv2/includes/tests.ts @@ -151,22 +151,29 @@ function templateAssertion( operator = "is"; } - lines.push( - `assert ${assertionValue} ${operator} ${templateValue( - field, - example, - false, - { - usageContext: usageContext, - operation: usageContext?.Operation, - test: usageContext.Test, - isTest: true, - typedDictDisabled: true, - isResponse: true, - targetingMockServer: usageContext.UsingMockServer, - }, - )}`, - ); + const expected = templateValue(field, example, false, { + usageContext: usageContext, + operation: usageContext?.Operation, + test: usageContext.Test, + isTest: true, + typedDictDisabled: true, + isResponse: true, + targetingMockServer: usageContext.UsingMockServer, + }); + + if ( + context.Global.Config.PreserveUnknownFields && + ["class", "array", "map", "union"].includes(field.Type.Type.toString()) + ) { + addImport(".unknown_fields", "drop_unexpected_fields"); + lines.push(`expected = ${expected}`); + lines.push( + `assert drop_unexpected_fields(${assertionValue}, expected) == expected`, + ); + return lines; + } + + lines.push(`assert ${assertionValue} ${operator} ${expected}`); return lines; } diff --git a/templates/templates/pythonv2/model.py.stmpl b/templates/templates/pythonv2/model.py.stmpl index 4e13fc018..6603486da 100644 --- a/templates/templates/pythonv2/model.py.stmpl +++ b/templates/templates/pythonv2/model.py.stmpl @@ -7,6 +7,7 @@ {{- $hasOptionalField := gt (len $optionalFields) 0 -}} {{- $hasNullableField := gt (len $nullableFields) 0 -}} {{- $hasAdditionalProperties := hasAdditionalPropertiesField $type -}} +{{- $preserveUnknownFields := .Global.Config.PreserveUnknownFields -}} {{- if $hasAdditionalProperties}}{{addImport "pydantic" "ConfigDict"}}{{end -}} {{- $fields := (sortFields $type.Fields) -}} {{- $className := sanitizeClassName $type.Name }} @@ -93,11 +94,20 @@ class {{$className}}(BaseModel): for n, f in type(self).model_fields.items(): k = f.alias if f.alias is not None else n + {{- if $preserveUnknownFields}} + {{- /* Pydantic writes extras after declared fields, so an extra under a field's emitted key replaces that field's value */}} + {{- /* A nested model is serialized again from its own output, which may already use aliases */}} + key = k if info.by_alias or k in serialized else n + if key not in serialized and not info.exclude_none: + continue + val = serialized.pop(key, None) + {{- else}} if k not in serialized and n not in serialized and not info.exclude_none: continue val = serialized.get(k, serialized.get(n)) - {{- if $hasAdditionalProperties}} + {{- if $hasAdditionalProperties}} serialized.pop(k, serialized.pop(n, None)) + {{- end}} {{- end}} {{- if and $hasOptionalField $hasNullableField}} @@ -126,7 +136,7 @@ class {{$className}}(BaseModel): m[k] = val {{- end}} - {{- if $hasAdditionalProperties}} + {{- if or $hasAdditionalProperties $preserveUnknownFields}} for k, v in serialized.items(): m[k] = v {{- end}} diff --git a/templates/templates/pythonv2/tests/common/unknown_fields_cases.py b/templates/templates/pythonv2/tests/common/unknown_fields_cases.py new file mode 100644 index 000000000..2d5aac5d6 --- /dev/null +++ b/templates/templates/pythonv2/tests/common/unknown_fields_cases.py @@ -0,0 +1,28 @@ +UNKNOWN_FIELD_CASES = [ + ({"futureKind": "third", "futureCode": 3}, {"futureKind": "third", "futureCode": 3}), + ({"future": "value"}, {}), + ({"future": False}, {}), + ({"empty": {"future": True}}, {"empty": {}}), + ({"future": None}, {}), + ({"plainChoice": {"text": "value", "future": True}}, {"plainChoice": {"text": "value"}}), + ({"plainChoice": {"number": 3, "future": True}}, {"plainChoice": {"number": 3}}), + ({"requiredOnly": {"label": "required", "future": True}}, {"requiredOnly": {"label": "required"}}), + ({"future": 0}, {}), + ({"composed": {"displayName": "child", "category": "combined", "future": True}}, {"composed": {"displayName": "child", "category": "combined"}}), + ({"split": {"label": "split", "future": True}}, {"split": {"label": "split"}}), + ({"future": []}, {}), + ({"future": {"nested": [None, False, 0, "value"]}}, {}), + ({"child": {"displayName": "child", "future": True}}, {"child": {"displayName": "child"}}), + ({"child": {"displayName": "child", "display_name": "unknown"}}, {"child": {"displayName": "child"}}), + ({"child": {"displayName": "child", "child": {"displayName": "grandchild", "future": True}}}, {"child": {"displayName": "child", "child": {"displayName": "grandchild"}}}), + ({"children": [{"displayName": "child", "future": True}]}, {"children": [{"displayName": "child"}]}), + ({"lookup": {"one": {"displayName": "child", "future": True}}}, {"lookup": {"one": {"displayName": "child"}}}), + ({"choice": {"kind": "text", "text": "value", "future": True}}, {"choice": {"kind": "text", "text": "value"}}), + ({"choice": {"kind": "number", "number": 3, "future": True}}, {"choice": {"kind": "number", "number": 3}}), + ({"closed": {"label": "closed", "future": True}}, {"closed": {"label": "closed"}}), + ({"open": {"label": "open", "future": True}}, {"open": {"label": "open", "future": True}}), + ({"typed": {"label": "typed", "future": "value"}}, {"typed": {"label": "typed", "future": "value"}}), + ({"typedNodes": {"label": "nodes", "one": {"displayName": "child", "future": True}}}, {"typedNodes": {"label": "nodes", "one": {"displayName": "child"}}}), + ({"arbitrary": {"future": {"nullable": None}}}, {"arbitrary": {"future": {"nullable": None}}}), + ({"child": {"displayName": "child", "nullable": None}}, {"child": {"displayName": "child", "nullable": None}}), +] diff --git a/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py new file mode 100644 index 000000000..a87a602a5 --- /dev/null +++ b/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py @@ -0,0 +1,23 @@ +import json + +import pytest + +from openapi import SDK, utils +from openapi.models import shared +from .common_helpers import HTTPBIN_URL +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +@pytest.mark.parametrize("fields, known_fields", UNKNOWN_FIELD_CASES) +def test_unknown_fields_dropped_by_default(fields, known_fields): + model = shared.FieldEnvelopeInput.model_validate({"label": "root", **fields}) + expected = {"label": "root", **known_fields} + assert model.model_dump(mode="json", by_alias=True, exclude_none=True) == expected + assert json.loads(utils.marshal_json(model, shared.FieldEnvelopeInput)) == expected + + +def test_flattened_request_drops_nested_unknown_fields_by_default(): + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_flattened_unknown_fields(label="root", child={"displayName": "child", "future": True}) + assert response.http_meta.response.json()["json"] == {"label": "root", "child": {"displayName": "child"}} + assert response.object.model_dump(by_alias=True, exclude_none=True) == {"json": {"label": "root", "child": {"displayName": "child"}}} diff --git a/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py new file mode 100644 index 000000000..a5927e445 --- /dev/null +++ b/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py @@ -0,0 +1,60 @@ +import pytest + +from openapi import SDK, AsyncSDK +from openapi.models import shared +from .common_helpers import HTTPBIN_URL, record_test +from .unknown_fields import drop_unexpected_fields + + +@pytest.mark.parametrize("child", [ + {"displayName": "child", "future": True}, + {"displayName": "child", "display_name": "unknown"}, + {"displayName": "child", "child": {"displayName": "grandchild", "future": {"nested": [None, 0]}}}, + {"displayName": "child", "nullable": None, "future": None}, +]) +def test_flattened_request_keeps_nested_unknown_fields(child): + record_test("unknown-fields-json-round-trip") + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_unknown_fields(label="root", child=child) + payload = {"label": "root", "child": child} + assert response.raw_response.json()["json"] == payload + assert response.object.json_.model_dump(mode="json", by_alias=True) == payload + + +@pytest.mark.asyncio() +async def test_flattened_request_keeps_nested_unknown_fields_async(): + child = {"displayName": "child", "child": {"displayName": "grandchild", "future": {"nested": [None, 0]}}} + async with AsyncSDK(server_url=HTTPBIN_URL) as sdk: + response = await sdk.echo_unknown_fields(label="root", child=child) + payload = {"label": "root", "child": child} + assert response.raw_response.json()["json"] == payload + assert response.object.json_.model_dump(mode="json", by_alias=True) == payload + + +def test_flattened_request_keeps_unknown_fields_on_nested_models(): + child = shared.FieldNode(display_name="child", future=None) + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_flattened_unknown_fields(label="root", child=child) + assert response.raw_response.json()["json"] == {"label": "root", "child": {"displayName": "child", "future": None}} + + +def test_flattened_request_has_no_argument_for_top_level_unknown_fields(): + with SDK(server_url=HTTPBIN_URL) as sdk: + with pytest.raises(TypeError): + sdk.echo_flattened_unknown_fields(label="root", future=True) # type: ignore[call-arg] + + +def test_drop_unexpected_fields_ignores_only_undeclared_fields(): + expected = shared.TypedFields(label="typed") + assert drop_unexpected_fields(shared.TypedFields.model_validate({"label": "typed", "future": "value"}), expected) != expected + assert drop_unexpected_fields(shared.FieldNode.model_validate({"displayName": "child", "future": True}), shared.FieldNode(display_name="child")) == shared.FieldNode(display_name="child") + + +def test_drop_unexpected_fields_in_typed_additional_properties(): + expected = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "child"}}) + nested_unknown = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "child", "future": True}}) + extra_entry = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "child"}, "two": {"displayName": "other"}}) + changed_value = shared.TypedNodes.model_validate({"label": "nodes", "one": {"displayName": "changed"}}) + assert drop_unexpected_fields(nested_unknown, expected) == expected + assert drop_unexpected_fields(extra_entry, expected) != expected + assert drop_unexpected_fields(changed_value, expected) != expected diff --git a/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py new file mode 100644 index 000000000..5b7c88ae8 --- /dev/null +++ b/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py @@ -0,0 +1,117 @@ +import json +from typing import Annotated, Any, List, Optional + +import httpx +import pydantic +import pytest + +from openapi import SDK, utils +from openapi.models import errors, shared +from openapi.types import UNSET, BaseModel, Nullable +from .common_helpers import HTTPBIN_URL, record_test +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +CASES = [fields for fields, _ in UNKNOWN_FIELD_CASES] + + +def echo(request): + sent = [] + with httpx.Client(event_hooks={"request": [sent.append]}) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + response = sdk.echo_unknown_fields(request=request) + return response, json.loads(sent[-1].content) + + +@pytest.mark.parametrize("fields", CASES) +@pytest.mark.parametrize("as_model", [False, True]) +def test_unknown_fields_echo(fields, as_model): + record_test("unknown-fields-json-round-trip") + payload = {"label": "root", **fields} + request = shared.FieldEnvelope.model_validate(payload) if as_model else payload + response, sent = echo(request) + assert sent == payload + assert response.json_.model_dump(mode="json", by_alias=True) == payload + assert json.loads(response.json_.model_dump_json(by_alias=True)) == payload + assert "headers" in response.model_dump() + + +def test_response_only_unknown_fields(): + payload = {"json": {"label": "root", "child": {"displayName": "child", "future": True}}, "future": {"value": 1}} + transport = httpx.MockTransport(lambda request: httpx.Response(200, json=payload)) + with httpx.Client(transport=transport) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + response = sdk.echo_unknown_fields(request={"label": "root"}) + dumped = response.model_dump(mode="json", by_alias=True) + assert "future" in dumped + assert "future" in dumped["json"]["child"] + + +def test_response_model_can_be_sent_again(): + payload = {"label": "root", "child": {"displayName": "child", "future": {"enabled": False}}} + first, _ = echo(payload) + second, sent = echo(first.json_.model_dump(by_alias=True)) + assert sent == payload + assert second.json_.model_dump(by_alias=True, exclude_none=True) == payload + + +def test_error_response_unknown_fields(): + payload = {"message": "invalid", "future": None, "detail": {"displayName": "child", "future": {"flag": True}}} + transport = httpx.MockTransport(lambda request: httpx.Response(400, json=payload)) + with httpx.Client(transport=transport) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + with pytest.raises(errors.EchoUnknownFieldsResponseBody) as caught: + sdk.echo_unknown_fields(request={"label": "root"}) + dumped = getattr(caught.value, "data").model_dump(mode="json", by_alias=True) + assert "future" in dumped + assert "future" in dumped["detail"] + + +def test_omitted_json_query_stays_empty(): + sent = [] + with httpx.Client(event_hooks={"request": [sent.append]}) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + sdk.echo_optional_json_query() + assert sent[-1].url.params["filter"] == "" + + +def test_typed_additional_properties_still_validate(): + with pytest.raises(pydantic.ValidationError): + shared.FieldEnvelope.model_validate({"label": "root", "typed": {"future": {"invalid": True}}}) + + +def test_unknown_fields_dump_filters(): + model = shared.FieldEnvelope.model_validate({"label": "root", "future": True, "future_null": None}) + assert "future_null" in model.model_dump() + assert "future_null" not in model.model_dump(exclude_none=True) + assert "future" not in model.model_dump(exclude={"future"}) + assert model.model_dump(include={"label"}) == {"label": "root"} + + +def test_base_model_unknown_field_policy(): + class CustomModel(BaseModel): + label: str + + model = CustomModel(label="value", future=None) + assert "future" in model.model_dump() + + +def test_invalid_nullable_value_becomes_unset_like_default_policy(): + node = shared.FieldNode.model_validate({"displayName": "child", "nullable": {"future": True}}) + assert node.nullable == UNSET + + +@pytest.mark.parametrize("typ", [Optional[int], Optional[List[int]], Optional[shared.FieldNode], Any]) +def test_top_level_none_serialization(typ): + assert utils.marshal_json(None, typ) == "" + + +def test_top_level_none_still_validates(): + with pytest.raises(pydantic.ValidationError): + utils.marshal_json(None, shared.FieldNode) + assert utils.marshal_json(None, Nullable[shared.FieldNode]) == "null" + + +def test_coerced_top_level_none_stays_empty(): + typ = Annotated[Optional[int], pydantic.BeforeValidator(lambda value: None)] + assert utils.marshal_json("empty", typ) == "" diff --git a/templates/templates/ruby/features.ts b/templates/templates/ruby/features.ts index 3435fe0de..fec3e49fc 100644 --- a/templates/templates/ruby/features.ts +++ b/templates/templates/ruby/features.ts @@ -475,6 +475,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/typescriptv2/features.ts b/templates/templates/typescriptv2/features.ts index becfb3d96..7a565f8eb 100644 --- a/templates/templates/typescriptv2/features.ts +++ b/templates/templates/typescriptv2/features.ts @@ -423,6 +423,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/unity/features.ts b/templates/templates/unity/features.ts index e69b828ac..4276f2af6 100644 --- a/templates/templates/unity/features.ts +++ b/templates/templates/unity/features.ts @@ -596,6 +596,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml b/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml index a61edf83b..98733ed2e 100644 --- a/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml @@ -35,6 +35,7 @@ python: responseFormat: envelope responseSchemaValidation: lenient flattenRequests: true + preserveUnknownFields: true pytestFilterWarnings: - error::RuntimeWarning # Ensure __del__ method warnings are raised diff --git a/tests/config/review/pythonv2/.speakeasy/gen.yaml b/tests/config/review/pythonv2/.speakeasy/gen.yaml index 7d4876daa..357bc40e1 100644 --- a/tests/config/review/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/review/pythonv2/.speakeasy/gen.yaml @@ -99,6 +99,7 @@ python: packageManager: uv packageName: openapi preApplyUnionDiscriminators: false + preserveUnknownFields: false pytestFilterWarnings: [] pytestTimeout: 0 rawResponseHelpers: false diff --git a/tests/config/secondary/pythonv2/.speakeasy/gen.yaml b/tests/config/secondary/pythonv2/.speakeasy/gen.yaml index 97ddd5bd2..fa854c67a 100644 --- a/tests/config/secondary/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/secondary/pythonv2/.speakeasy/gen.yaml @@ -38,6 +38,7 @@ python: webhooks: models/webhooks maxMethodParams: 5 flattenRequests: false + preserveUnknownFields: true packageName: openapi rawResponseHelpers: true responseFormat: flat diff --git a/tests/overlays/tertiary/typescriptv2/overlay.yaml b/tests/overlays/tertiary/typescriptv2/overlay.yaml new file mode 100644 index 000000000..fcc524d36 --- /dev/null +++ b/tests/overlays/tertiary/typescriptv2/overlay.yaml @@ -0,0 +1,9 @@ +overlay: 1.0.0 +info: + title: Overlay ./tests/specs/uber.yaml => tertiary typescriptv2 test spec + version: 0.0.0 +actions: + # Inline success and error bodies share a generated name, which collides on import + # without index modules, flat responses and the legacy naming scheme + - target: $.paths.*.*[?(@.operationId == 'echoUnknownFields')] + remove: true diff --git a/tests/specs/fragments/uber/unknown-fields.yaml b/tests/specs/fragments/uber/unknown-fields.yaml new file mode 100644 index 000000000..8ed2b359c --- /dev/null +++ b/tests/specs/fragments/uber/unknown-fields.yaml @@ -0,0 +1,256 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +paths: + /anything/unknown-fields-query: + get: + operationId: echoOptionalJsonQuery + description: Echoes an optional JSON query parameter without changing its omission semantics. + parameters: + - name: filter + in: query + required: false + content: + application/json: + schema: + title: UnknownFieldsQueryFilter + type: object + properties: + label: + type: string + responses: + '200': + description: The echoed request URL. + content: + application/json: + schema: + type: object + properties: + url: + type: string + /anything/unknown-fields: + post: + operationId: echoUnknownFields + x-speakeasy-exclude-test-sdk: [javav2] + description: Echoes nested objects to exercise preservation of undeclared fields in requests and responses. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FieldEnvelope' + responses: + '200': + description: The echoed JSON request and transport metadata. + content: + application/json: + schema: + type: object + required: [json] + properties: + json: + $ref: '#/components/schemas/FieldEnvelope' + '400': + description: A structured error with additional server fields. + content: + application/json: + schema: + type: object + required: [message] + properties: + message: + type: string + detail: + $ref: '#/components/schemas/FieldNode' + /anything/unknown-fields-flattened: + post: + operationId: echoFlattenedUnknownFields + description: Echoes a body small enough to be flattened into operation arguments, so undeclared fields can only travel in nested objects. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FlattenedFields' + responses: + '200': + description: The echoed JSON request. + content: + application/json: + schema: + type: object + required: [json] + properties: + json: + $ref: '#/components/schemas/FlattenedFields' +components: + schemas: + FieldEnvelope: + type: object + required: [label] + properties: + label: + type: string + child: + $ref: '#/components/schemas/FieldNode' + children: + type: array + items: + $ref: '#/components/schemas/FieldNode' + lookup: + type: object + additionalProperties: + $ref: '#/components/schemas/FieldNode' + choice: + oneOf: + - $ref: '#/components/schemas/TextChoice' + - $ref: '#/components/schemas/NumberChoice' + discriminator: + propertyName: kind + mapping: + text: '#/components/schemas/TextChoice' + number: '#/components/schemas/NumberChoice' + plainChoice: + anyOf: + - $ref: '#/components/schemas/TextFields' + - $ref: '#/components/schemas/NumberFields' + requiredOnly: + $ref: '#/components/schemas/RequiredFields' + composed: + allOf: + - $ref: '#/components/schemas/FieldNode' + - type: object + properties: + category: + type: string + split: + $ref: '#/components/schemas/SplitFields' + empty: + $ref: '#/components/schemas/EmptyFields' + open: + $ref: '#/components/schemas/OpenFields' + typed: + $ref: '#/components/schemas/TypedFields' + typedNodes: + $ref: '#/components/schemas/TypedNodes' + closed: + $ref: '#/components/schemas/ClosedFields' + arbitrary: {} + futureKind: + type: string + enum: [first, second] + x-speakeasy-unknown-values: allow + futureCode: + type: integer + enum: [1, 2] + x-speakeasy-unknown-values: allow + FieldNode: + type: object + required: [displayName] + properties: + displayName: + type: string + nullable: + type: [string, 'null'] + child: + $ref: '#/components/schemas/FieldNode' + TextChoice: + type: object + required: [kind, text] + properties: + kind: + $ref: '#/components/schemas/TextChoiceKind' + text: + type: string + NumberChoice: + type: object + required: [kind, number] + properties: + kind: + $ref: '#/components/schemas/NumberChoiceKind' + number: + type: integer + TextChoiceKind: + type: string + enum: [text] + NumberChoiceKind: + type: string + enum: [number] + OpenFields: + type: object + additionalProperties: true + properties: + label: + type: string + TypedFields: + type: object + additionalProperties: + type: string + properties: + label: + type: string + TypedNodes: + type: object + description: Additional properties typed as an object schema, whose values can carry undeclared fields of their own. + additionalProperties: + $ref: '#/components/schemas/FieldNode' + properties: + label: + type: string + ClosedFields: + type: object + additionalProperties: false + properties: + label: + type: string + TextFields: + type: object + required: [text] + properties: + text: + type: string + label: + type: string + NumberFields: + type: object + required: [number] + properties: + number: + type: integer + label: + type: string + RequiredFields: + type: object + required: [label] + properties: + label: + type: string + SplitFields: + type: object + properties: + label: + type: string + id: + type: string + readOnly: true + secret: + type: string + writeOnly: true + EmptyFields: + type: object + additionalProperties: false + FlattenedFields: + type: object + description: A body with few enough properties to be flattened into operation arguments. + required: [label] + properties: + label: + type: string + child: + $ref: '#/components/schemas/FieldNode' diff --git a/zSDKs/sdk-pythonv2/.speakeasy/gen.lock b/zSDKs/sdk-pythonv2/.speakeasy/gen.lock index e5217c27c..4dd87f1bf 100644 --- a/zSDKs/sdk-pythonv2/.speakeasy/gen.lock +++ b/zSDKs/sdk-pythonv2/.speakeasy/gen.lock @@ -6,7 +6,7 @@ management: speakeasyVersion: internal generationVersion: internal releaseVersion: 0.0.1 - configChecksum: d67daf2ec44e985625b98d32573578c6 + configChecksum: 908ffac3f2935a4cd93222913042d257 repoURL: https://github.com/speakeasy-sdks/test-sdk installationURL: https://github.com/speakeasy-sdks/test-sdk.git features: @@ -746,7 +746,7 @@ trackedFiles: src/speakeasy/new_openapi/types/base64fileinput.py: last_write_checksum: sha1:1522687ae3398374c35710cad993a6e82b5ab99d src/speakeasy/new_openapi/types/basemodel.py: - last_write_checksum: sha1:6295cc5750e00e0323902092f7e52549937b0b9c + last_write_checksum: sha1:16e62753ef3b7f377dd826eaff379a519e1096d3 src/speakeasy/new_openapi/types_.py: last_write_checksum: sha1:05e36f6428f04b1ea7d2609aa55f3966868b58f0 src/speakeasy/new_openapi/utils/__init__.py: diff --git a/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml b/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml index 7d4876daa..357bc40e1 100644 --- a/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml +++ b/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml @@ -99,6 +99,7 @@ python: packageManager: uv packageName: openapi preApplyUnionDiscriminators: false + preserveUnknownFields: false pytestFilterWarnings: [] pytestTimeout: 0 rawResponseHelpers: false diff --git a/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py b/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py index 06c13bfcd..5c4819c8b 100644 --- a/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py +++ b/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py @@ -37,7 +37,9 @@ def __bool__(self) -> Literal[False]: ) -def _unrecognized_serialization(cls: Type[Union[str, int]]) -> core_schema.PlainSerializerFunctionSerSchema: +def _unrecognized_serialization( + cls: Type[Union[str, int]], +) -> core_schema.PlainSerializerFunctionSerSchema: def serialize(value: Any, info: core_schema.SerializationInfo) -> Any: # Decline other values so union serialization tries the remaining members if not isinstance(value, cls): @@ -51,10 +53,12 @@ def serialize(value: Any, info: core_schema.SerializationInfo) -> Any: class UnrecognizedStr(str): @classmethod - def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> core_schema.CoreSchema: + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: Any + ) -> core_schema.CoreSchema: # Make UnrecognizedStr only work in lax mode, not strict mode # This makes it a "fallback" option when more specific types (like Literals) don't match - def validate_lax(v: Any) -> 'UnrecognizedStr': + def validate_lax(v: Any) -> "UnrecognizedStr": if isinstance(v, cls): return v return cls(str(v)) @@ -63,10 +67,12 @@ def validate_lax(v: Any) -> 'UnrecognizedStr': # This forces Pydantic to prefer other union members in strict mode # and only fall back to UnrecognizedStr in lax mode return core_schema.lax_or_strict_schema( - lax_schema=core_schema.chain_schema([ - core_schema.str_schema(), - core_schema.no_info_plain_validator_function(validate_lax) - ]), + lax_schema=core_schema.chain_schema( + [ + core_schema.str_schema(), + core_schema.no_info_plain_validator_function(validate_lax), + ] + ), strict_schema=core_schema.none_schema(), # Always fails in strict mode serialization=_unrecognized_serialization(cls), ) @@ -74,18 +80,23 @@ def validate_lax(v: Any) -> 'UnrecognizedStr': class UnrecognizedInt(int): @classmethod - def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> core_schema.CoreSchema: + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: Any + ) -> core_schema.CoreSchema: # Make UnrecognizedInt only work in lax mode, not strict mode # This makes it a "fallback" option when more specific types (like Literals) don't match - def validate_lax(v: Any) -> 'UnrecognizedInt': + def validate_lax(v: Any) -> "UnrecognizedInt": if isinstance(v, cls): return v return cls(int(v)) + return core_schema.lax_or_strict_schema( - lax_schema=core_schema.chain_schema([ - core_schema.int_schema(), - core_schema.no_info_plain_validator_function(validate_lax) - ]), + lax_schema=core_schema.chain_schema( + [ + core_schema.int_schema(), + core_schema.no_info_plain_validator_function(validate_lax), + ] + ), strict_schema=core_schema.none_schema(), # Always fails in strict mode serialization=_unrecognized_serialization(cls), )