diff --git a/.changesets/1792668682-0165685a.yaml b/.changesets/1792668682-0165685a.yaml new file mode 100644 index 000000000..1ec90ee6b --- /dev/null +++ b/.changesets/1792668682-0165685a.yaml @@ -0,0 +1,10 @@ +id: 1792668682-0165685a +features: + - core +targets: + - pythonv2 +type: feat +bump: patch +description: add opt-in model serialization helpers and unknown-field preservation; fix open enum JSON serialization +author: ThomasRooney +date: "2026-10-22" diff --git a/internal/features/tests.go b/internal/features/tests.go index a4031fcb5..c936b8b3f 100644 --- a/internal/features/tests.go +++ b/internal/features/tests.go @@ -781,6 +781,7 @@ const ( TestJsonlStreamTimeoutStillBoundsSlowStream TestJsonlStreamWithTimeoutStreamsToCompletion TestSharedEnumConstructorDefault + TestUnknownFieldsJsonRoundTrip ) func GetAllTests() []Test { diff --git a/internal/features/tests_generated.go b/internal/features/tests_generated.go index b6dd807de..01446360c 100644 --- a/internal/features/tests_generated.go +++ b/internal/features/tests_generated.go @@ -1502,6 +1502,8 @@ func (t Test) String() string { return "unions-weakly-typed-one-of-post-basic" case TestUnionsWeaklyTypedOneOfPostDeep: return "unions-weakly-typed-one-of-post-deep" + case TestUnknownFieldsJsonRoundTrip: + return "unknown-fields-json-round-trip" case TestWebhooksConsume: return "webhooks-consume" case TestWebhooksConsumeBadData: @@ -3022,6 +3024,8 @@ func TestFromString(s string) Test { return TestUnionsWeaklyTypedOneOfPostBasic case "unions-weakly-typed-one-of-post-deep": return TestUnionsWeaklyTypedOneOfPostDeep + case "unknown-fields-json-round-trip": + return TestUnknownFieldsJsonRoundTrip case "webhooks-consume": return TestWebhooksConsume case "webhooks-consume-bad-data": @@ -3792,6 +3796,7 @@ var testList = []Test{ TestUnionsUnionOfArrays, TestUnionsWeaklyTypedOneOfPostBasic, TestUnionsWeaklyTypedOneOfPostDeep, + TestUnknownFieldsJsonRoundTrip, TestWebhooksConsume, TestWebhooksConsumeBadData, TestWebhooksConsumeBadSignature, diff --git a/templates/perms.go b/templates/perms.go index 3bd7b0c7e..15d73e517 100644 --- a/templates/perms.go +++ b/templates/perms.go @@ -1543,6 +1543,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/auxiliary/tests/conftest.py.stmpl": 0o644, "templates/pythonv2/auxiliary/tests/helpers.py.stmpl": 0o644, "templates/pythonv2/auxiliary/tests/test_client.py.stmpl": 0o644, + "templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/__init__.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/_version.py.stmpl": 0o644, @@ -1552,7 +1553,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/auxiliary/{{getSourcePath}}/types": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/types/__init__.py": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/types/base64fileinput.py": 0o644, - "templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py": 0o644, + "templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils": 0o20000000755, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils/__init__.py.stmpl": 0o644, "templates/pythonv2/auxiliary/{{getSourcePath}}/utils/annotations.py": 0o644, @@ -1668,6 +1669,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/client-credentials/test_client_credentials_hook.py": 0o644, "templates/pythonv2/tests/common": 0o20000000755, "templates/pythonv2/tests/common/common_helpers.py": 0o644, + "templates/pythonv2/tests/common/unknown_fields_cases.py": 0o644, "templates/pythonv2/tests/custom-http": 0o20000000755, "templates/pythonv2/tests/custom-http/test_auth_additional.py": 0o644, "templates/pythonv2/tests/oauth2-password": 0o20000000755, @@ -1702,6 +1704,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/primary/test_telemetry.py.stmpl": 0o644, "templates/pythonv2/tests/primary/test_timeout_additional.py": 0o644, "templates/pythonv2/tests/primary/test_unions_additional.py": 0o644, + "templates/pythonv2/tests/primary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/primary/test_validation_errors.py": 0o644, "templates/pythonv2/tests/primary/test_x_ndjson_additional.py": 0o644, "templates/pythonv2/tests/quaternary": 0o20000000755, @@ -1711,6 +1714,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/quaternary/test_globals_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_jsonl_streaming_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_servers.py": 0o644, + "templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/quaternary/test_validation_errors.py": 0o644, "templates/pythonv2/tests/secondary": 0o20000000755, "templates/pythonv2/tests/secondary/test_auth_additional.py": 0o644, @@ -1725,6 +1729,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/secondary/test_pagination.py": 0o644, "templates/pythonv2/tests/secondary/test_redirects_additional.py": 0o644, "templates/pythonv2/tests/secondary/test_servers.py": 0o644, + "templates/pythonv2/tests/secondary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/tests/security-options": 0o20000000755, "templates/pythonv2/tests/security-options/test_auth_additional.py": 0o644, "templates/pythonv2/tests/tertiary": 0o20000000755, @@ -1732,6 +1737,7 @@ var fileMode = map[string]fs.FileMode{ "templates/pythonv2/tests/tertiary/test_body_typeddict_suffix.py": 0o644, "templates/pythonv2/tests/tertiary/test_relative_imports.py": 0o644, "templates/pythonv2/tests/tertiary/test_servers.py": 0o644, + "templates/pythonv2/tests/tertiary/test_unknown_fields_additional.py": 0o644, "templates/pythonv2/testusage": 0o20000000755, "templates/pythonv2/testusage/pylintrc": 0o644, "templates/pythonv2/testusage/pyproject.toml.stmpl": 0o644, diff --git a/templates/templates/cli/features.ts b/templates/templates/cli/features.ts index a0c382e38..6cbf8cec5 100644 --- a/templates/templates/cli/features.ts +++ b/templates/templates/cli/features.ts @@ -359,6 +359,7 @@ function isTestSkipped(test: string): boolean { "parameters-path-parameter-formats", "shared-enum-constructor-default", "open-union-invalid-json-is-an-error", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/csharp/features.ts b/templates/templates/csharp/features.ts index 8c3bd71eb..6f8672df7 100644 --- a/templates/templates/csharp/features.ts +++ b/templates/templates/csharp/features.ts @@ -430,6 +430,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/go/features.ts b/templates/templates/go/features.ts index 1283baf4b..a82334d3c 100644 --- a/templates/templates/go/features.ts +++ b/templates/templates/go/features.ts @@ -427,6 +427,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/javav2/features.ts b/templates/templates/javav2/features.ts index f9d5e49b3..47b5f3f9a 100644 --- a/templates/templates/javav2/features.ts +++ b/templates/templates/javav2/features.ts @@ -478,6 +478,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/mcp-typescript/features.ts b/templates/templates/mcp-typescript/features.ts index 70a1d2124..66dc387a4 100644 --- a/templates/templates/mcp-typescript/features.ts +++ b/templates/templates/mcp-typescript/features.ts @@ -687,6 +687,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/php/features.ts b/templates/templates/php/features.ts index 0254cdade..3c421743e 100644 --- a/templates/templates/php/features.ts +++ b/templates/templates/php/features.ts @@ -572,6 +572,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl b/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl new file mode 100644 index 000000000..467f8f68b --- /dev/null +++ b/templates/templates/pythonv2/auxiliary/tests/unknown_fields.py.stmpl @@ -0,0 +1,31 @@ +{{- if and (sdkHasTests .Global.AST) .Global.Config.PreserveUnknownFields -}} +from typing import Any + +from pydantic import BaseModel + + +def drop_unexpected_fields(actual: Any, expected: Any) -> Any: + if isinstance(actual, BaseModel) and type(actual) is type(expected): + trimmed = actual.model_copy( + update={ + name: drop_unexpected_fields(getattr(actual, name), getattr(expected, name)) + for name in type(actual).model_fields + } + ) + if actual.__pydantic_extra__ is not None and "__pydantic_extra__" not in type(actual).__annotations__: + expected_extra = expected.__pydantic_extra__ or {} + trimmed.__pydantic_extra__ = { + key: drop_unexpected_fields(value, expected_extra[key]) + for key, value in actual.__pydantic_extra__.items() + if key in expected_extra + } + return trimmed + if isinstance(actual, list) and isinstance(expected, list) and len(actual) == len(expected): + return [drop_unexpected_fields(a, e) for a, e in zip(actual, expected)] + if isinstance(actual, dict) and isinstance(expected, dict): + return { + key: drop_unexpected_fields(value, expected[key]) if key in expected else value + for key, value in actual.items() + } + return actual +{{- end}} diff --git a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl similarity index 52% rename from templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py rename to templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl index d4744c6eb..6fd3a4868 100644 --- a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py +++ b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/types/basemodel.py.stmpl @@ -1,17 +1,62 @@ from pydantic import ConfigDict, model_serializer from pydantic import BaseModel as PydanticBaseModel -from pydantic_core import core_schema -from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union +from pydantic_core import PydanticSerializationUnexpectedValue, core_schema +from typing import TYPE_CHECKING, Any, Literal, Optional, Type, TypeVar, Union from typing_extensions import TypeAliasType, TypeAlias class BaseModel(PydanticBaseModel): model_config = ConfigDict( - populate_by_name=True, arbitrary_types_allowed=True, protected_namespaces=() + populate_by_name=True, arbitrary_types_allowed=True, protected_namespaces=(){{if .Global.Config.PreserveUnknownFields}}, extra="allow"{{end}} ) +{{if .Global.Config.ModelSerializationMethods}} + def to_dict( + self, + *, + mode: Literal["json", "python"] = "python", + use_api_names: bool = True, + exclude_unset: bool = True, + exclude_defaults: bool = False, + exclude_none: bool = False, + warnings: bool = True, + ) -> dict[str, Any]: + return self.model_dump( + mode=mode, + by_alias=use_api_names, + exclude_unset=exclude_unset, + exclude_defaults=exclude_defaults, + exclude_none=exclude_none, + warnings=warnings, + context={SERIALIZATION_METHODS_DUMP: True}, + ) + + def to_json( + self, + *, + indent: Optional[int] = 2, + use_api_names: bool = True, + exclude_unset: bool = True, + exclude_defaults: bool = False, + exclude_none: bool = False, + warnings: bool = True, + ) -> str: + return self.model_dump_json( + indent=indent, + by_alias=use_api_names, + exclude_unset=exclude_unset, + exclude_defaults=exclude_defaults, + exclude_none=exclude_none, + warnings=warnings, + context={SERIALIZATION_METHODS_DUMP: True}, + ) + +{{end}} class Unset(BaseModel): +{{- if .Global.Config.PreserveUnknownFields}} + model_config = ConfigDict(extra="ignore") +{{end}} @model_serializer(mode="plain") def serialize_model(self): return UNSET_SENTINEL @@ -22,6 +67,9 @@ def __bool__(self) -> Literal[False]: UNSET = Unset() UNSET_SENTINEL = "~?~unset~?~sentinel~?~" +{{- if .Global.Config.ModelSerializationMethods}} +SERIALIZATION_METHODS_DUMP = "speakeasy_serialization_methods_dump" +{{- end}} T = TypeVar("T") @@ -35,6 +83,18 @@ def __bool__(self) -> Literal[False]: ) +def _unrecognized_serialization(cls: Type[Union[str, int]]) -> core_schema.PlainSerializerFunctionSerSchema: + def serialize(value: Any, info: core_schema.SerializationInfo) -> Any: + # Decline other values so union serialization tries the remaining members + if not isinstance(value, cls): + raise PydanticSerializationUnexpectedValue(f"Expected {cls.__name__}") + if info.mode == "python": + return value + return str(value) if isinstance(value, str) else int(value) + + return core_schema.plain_serializer_function_ser_schema(serialize, info_arg=True) + + class UnrecognizedStr(str): @classmethod def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> core_schema.CoreSchema: @@ -54,6 +114,7 @@ def validate_lax(v: Any) -> 'UnrecognizedStr': core_schema.no_info_plain_validator_function(validate_lax) ]), strict_schema=core_schema.none_schema(), # Always fails in strict mode + serialization=_unrecognized_serialization(cls), ) @@ -72,4 +133,5 @@ def validate_lax(v: Any) -> 'UnrecognizedInt': core_schema.no_info_plain_validator_function(validate_lax) ]), strict_schema=core_schema.none_schema(), # Always fails in strict mode + serialization=_unrecognized_serialization(cls), ) diff --git a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl index a3ea844b5..2062de6bb 100644 --- a/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl +++ b/templates/templates/pythonv2/auxiliary/{{getSourcePath}}/utils/serializers.py.stmpl @@ -278,7 +278,7 @@ def marshal_json(val, typ): m = marshaller(body=val) - d = m.model_dump(by_alias=True, mode="json", exclude_none=True) + d = m.model_dump(by_alias=True, mode="json", exclude_none={{if .Global.Config.PreserveUnknownFields}}getattr(m, "body") is None{{else}}True{{end}}) if len(d) == 0: return "" diff --git a/templates/templates/pythonv2/config.ts b/templates/templates/pythonv2/config.ts index e441cf95b..8582264c6 100644 --- a/templates/templates/pythonv2/config.ts +++ b/templates/templates/pythonv2/config.ts @@ -325,6 +325,24 @@ function getConfigFields( ValidationRegex: /^(true|false)$/.source, ValidationMessage: "true or false only", }, + modelSerializationMethods: { + Name: "modelSerializationMethods", + Required: false, + DefaultValue: false, + Description: + "Add to_dict and to_json convenience methods to models, using API field names and excluding unset fields by default.", + ValidationRegex: /^(true|false)$/.source, + ValidationMessage: "true or false only", + }, + preserveUnknownFields: { + Name: "preserveUnknownFields", + Required: false, + DefaultValue: false, + Description: + "Preserve undeclared fields in JSON request and response models, including nested models and schemas with additionalProperties: false. Explicitly typed additional properties retain their validation. Flattened operation arguments accept only declared fields, so undeclared top-level request fields require flattenRequests: false; nested models passed as arguments keep theirs. Disabled by default.", + ValidationRegex: /^(true|false)$/.source, + ValidationMessage: "true or false only", + }, responseSchemaValidation: { Name: "responseSchemaValidation", Required: false, diff --git a/templates/templates/pythonv2/includes/sanitization.ts b/templates/templates/pythonv2/includes/sanitization.ts index 74309517a..929808871 100644 --- a/templates/templates/pythonv2/includes/sanitization.ts +++ b/templates/templates/pythonv2/includes/sanitization.ts @@ -257,12 +257,12 @@ function sanitizeSDKAccess(sdk: SDK): string { let formattedName = ""; if (group.includes(".") && ".".repeat(group.length) !== group) { const pieces = group.split("."); - formattedName = sanitizeFieldName(pieces[0]); + formattedName = sanitizeSubSDKFieldName(pieces[0]); for (const i of pieces.slice(1)) { - formattedName += "." + sanitizeFieldName(i); + formattedName += "." + sanitizeSubSDKFieldName(i); } } else { - formattedName = sanitizeFieldName(group); + formattedName = sanitizeSubSDKFieldName(group); } if (formattedName != "") { @@ -311,6 +311,20 @@ registerTemplateFunc("sanitizeSDKFileName", sanitizeSDKFileName); // @ts-ignore function sanitizeFieldName(name: string): string { + name = sanitizeAttributeName(name); + + if ( + context.Global.Config.ModelSerializationMethods && + ["to_dict", "to_json"].includes(name) + ) { + name += "_"; + } + + return name; +} +registerTemplateFunc("sanitizeFieldName", sanitizeFieldName); + +function sanitizeAttributeName(name: string): string { name = sanitizeName(name); if (name.endsWith("_")) { name = name.substring(0, name.length - 1); @@ -326,7 +340,6 @@ function sanitizeFieldName(name: string): string { return name; } -registerTemplateFunc("sanitizeFieldName", sanitizeFieldName); function useConflictResistantModelImports(): boolean { return !!context.Global.Config.FixFlags?.[ @@ -413,9 +426,7 @@ function doesModuleNameConflictAcrossNamespaces( // @ts-ignore function sanitizeSubSDKFieldName(name: string): string { - name = sanitizeFieldName(name); - - return name; + return sanitizeAttributeName(name); } registerTemplateFunc("sanitizeSubSDKFieldName", sanitizeSubSDKFieldName); diff --git a/templates/templates/pythonv2/includes/templating.ts b/templates/templates/pythonv2/includes/templating.ts index 4ed12dd07..9b006ef2a 100644 --- a/templates/templates/pythonv2/includes/templating.ts +++ b/templates/templates/pythonv2/includes/templating.ts @@ -922,7 +922,7 @@ function templateSDKInitFieldName(name: string, suffix: string): string { reserved.push("server_idx"); } - let fieldName = sanitizeFieldName(name); + let fieldName = sanitizeAttributeName(name); if (reserved.includes(name)) { fieldName += "_" + suffix; @@ -1096,7 +1096,12 @@ function templateUsageSDKParams(local: UsageContext): string { return; } - params.push(`${sanitizeFieldName(parameter.field.Name)}=${value},`); + params.push( + `${templateSDKInitFieldName( + parameter.field.Name, + "global", + )}=${value},`, + ); break; } } @@ -1850,6 +1855,24 @@ function templateOptionalFieldList(typeDef: TypeDef): string { } registerTemplateFunc("templateOptionalFieldList", templateOptionalFieldList); +function templateConstFieldAttributeList(typeDef: TypeDef): string { + return typeDef.Fields.filter( + (f) => + f.Const && + !(f.Optional && f.Const.Value === null) && + !f.IsAdditionalProperties, + ) + .map((f) => { + const fieldName = sanitizeFieldName(f.Name); + return `"${useUpperCaseConsts() ? fieldName.toUpperCase() : fieldName}"`; + }) + .join(", "); +} +registerTemplateFunc( + "templateConstFieldAttributeList", + templateConstFieldAttributeList, +); + function templateNullableFieldList(typeDef: TypeDef): string { return templateFieldList(typeDef, (f) => f.Nullable); } diff --git a/templates/templates/pythonv2/includes/tests.ts b/templates/templates/pythonv2/includes/tests.ts index 7a5a2de87..9da99b1ac 100644 --- a/templates/templates/pythonv2/includes/tests.ts +++ b/templates/templates/pythonv2/includes/tests.ts @@ -151,22 +151,29 @@ function templateAssertion( operator = "is"; } - lines.push( - `assert ${assertionValue} ${operator} ${templateValue( - field, - example, - false, - { - usageContext: usageContext, - operation: usageContext?.Operation, - test: usageContext.Test, - isTest: true, - typedDictDisabled: true, - isResponse: true, - targetingMockServer: usageContext.UsingMockServer, - }, - )}`, - ); + const expected = templateValue(field, example, false, { + usageContext: usageContext, + operation: usageContext?.Operation, + test: usageContext.Test, + isTest: true, + typedDictDisabled: true, + isResponse: true, + targetingMockServer: usageContext.UsingMockServer, + }); + + if ( + context.Global.Config.PreserveUnknownFields && + ["class", "array", "map", "union"].includes(field.Type.Type.toString()) + ) { + addImport(".unknown_fields", "drop_unexpected_fields"); + lines.push(`expected = ${expected}`); + lines.push( + `assert drop_unexpected_fields(${assertionValue}, expected) == expected`, + ); + return lines; + } + + lines.push(`assert ${assertionValue} ${operator} ${expected}`); return lines; } diff --git a/templates/templates/pythonv2/model.py.stmpl b/templates/templates/pythonv2/model.py.stmpl index c4d42bc95..ce2608017 100644 --- a/templates/templates/pythonv2/model.py.stmpl +++ b/templates/templates/pythonv2/model.py.stmpl @@ -7,6 +7,11 @@ {{- $hasOptionalField := gt (len $optionalFields) 0 -}} {{- $hasNullableField := gt (len $nullableFields) 0 -}} {{- $hasAdditionalProperties := hasAdditionalPropertiesField $type -}} +{{- $preserveUnknownFields := .Global.Config.PreserveUnknownFields -}} +{{- $modelSerializationMethods := .Global.Config.ModelSerializationMethods -}} +{{- $serializerUsesInfo := or $preserveUnknownFields $modelSerializationMethods -}} +{{- $outputKey := "k" -}} +{{- if $modelSerializationMethods}}{{$outputKey = "n if use_field_names else k"}}{{end -}} {{- if $hasAdditionalProperties}}{{addImport "pydantic" "ConfigDict"}}{{end -}} {{- $fields := (sortFields $type.Fields) -}} {{- $className := sanitizeClassName $type.Name }} @@ -56,6 +61,14 @@ class {{$className}}(BaseModel): self.__pydantic_extra__ = value # pyright: ignore[reportIncompatibleVariableOverride] {{end}} + {{- $constFieldAttributes := templateConstFieldAttributeList $type}} + {{- if and .Global.Config.ModelSerializationMethods $constFieldAttributes}} + {{- addImport "typing" "Any"}} + + def model_post_init(self, context: Any, /) -> None: + self.__pydantic_fields_set__.update([{{$constFieldAttributes}}]) + {{- end}} + {{- if hasOpenEnumField $type}} {{- range openEnums $type}} {{- if and (not .Const) (not (eq (getEnumFormat .Type) "union"))}} @@ -76,7 +89,7 @@ class {{$className}}(BaseModel): {{- addImport "pydantic" "model_serializer"}} {{- addImport "types" "UNSET_SENTINEL" true}} @model_serializer(mode="wrap") - def {{sanitizeSerializeMethodName $type}}(self, handler): + def {{sanitizeSerializeMethodName $type}}(self, handler{{if $serializerUsesInfo}}, info{{end}}): {{- $nullDefaultFields := templateNullDefaultFieldList $type}} {{- $hasNullDefaultField := gt (len $nullDefaultFields) 0}} {{- if $hasOptionalField}} @@ -88,14 +101,31 @@ class {{$className}}(BaseModel): {{- end}} {{- end}} {{- end}} + {{- /* Pydantic writes extras after declared fields, so an extra under a field's emitted key replaces that field's value */}} serialized = handler(self) m = {} + {{- if $modelSerializationMethods}} + {{- addImport "types.basemodel" "SERIALIZATION_METHODS_DUMP" true}} + use_field_names = info.by_alias is False + serialization_methods_dump = isinstance(info.context, dict) and SERIALIZATION_METHODS_DUMP in info.context + {{- end}} for n, f in type(self).model_fields.items(): k = f.alias or n + {{- if $serializerUsesInfo}} + key = k if info.by_alias else n + {{- end}} + {{- if $modelSerializationMethods}} + if key not in serialized and (serialization_methods_dump or not info.exclude_none): + continue + {{- end}} + {{- if $serializerUsesInfo}} + val = serialized.pop(key, None) + {{- else}} val = serialized.get(k, serialized.get(n)) - {{- if $hasAdditionalProperties}} + {{- if $hasAdditionalProperties}} serialized.pop(k, serialized.pop(n, None)) + {{- end}} {{- end}} {{- if and $hasOptionalField $hasNullableField}} @@ -112,7 +142,7 @@ class {{$className}}(BaseModel): if val != UNSET_SENTINEL: {{- if not $hasOptionalField}} - m[k] = val + m[{{$outputKey}}] = val {{- else }} if ( val is not None @@ -121,10 +151,10 @@ class {{$className}}(BaseModel): or is_nullable_and_explicitly_set {{- end}} ): - m[k] = val + m[{{$outputKey}}] = val {{- end}} - {{- if $hasAdditionalProperties}} + {{- if or $hasAdditionalProperties $preserveUnknownFields}} for k, v in serialized.items(): m[k] = v {{- end}} diff --git a/templates/templates/pythonv2/tests/common/unknown_fields_cases.py b/templates/templates/pythonv2/tests/common/unknown_fields_cases.py new file mode 100644 index 000000000..5c8dc447b --- /dev/null +++ b/templates/templates/pythonv2/tests/common/unknown_fields_cases.py @@ -0,0 +1,28 @@ +UNKNOWN_FIELD_CASES = [ + ({"futureKind": "third", "futureCode": 3}, {"futureKind": "third", "futureCode": 3}), + ({"methodNames": {"toDict": "dictionary", "toJson": "json"}}, {"methodNames": {"toDict": "dictionary", "toJson": "json"}}), + ({"future": "value"}, {}), + ({"future": False}, {}), + ({"empty": {"future": True}}, {"empty": {}}), + ({"future": None}, {}), + ({"plainChoice": {"text": "value", "future": True}}, {"plainChoice": {"text": "value"}}), + ({"plainChoice": {"number": 3, "future": True}}, {"plainChoice": {"number": 3}}), + ({"requiredOnly": {"label": "required", "future": True}}, {"requiredOnly": {"label": "required"}}), + ({"future": 0}, {}), + ({"composed": {"displayName": "child", "category": "combined", "future": True}}, {"composed": {"displayName": "child", "category": "combined"}}), + ({"split": {"label": "split", "future": True}}, {"split": {"label": "split"}}), + ({"future": []}, {}), + ({"future": {"nested": [None, False, 0, "value"]}}, {}), + ({"child": {"displayName": "child", "future": True}}, {"child": {"displayName": "child"}}), + ({"child": {"displayName": "child", "display_name": "unknown"}}, {"child": {"displayName": "child"}}), + ({"child": {"displayName": "child", "child": {"displayName": "grandchild", "future": True}}}, {"child": {"displayName": "child", "child": {"displayName": "grandchild"}}}), + ({"children": [{"displayName": "child", "future": True}]}, {"children": [{"displayName": "child"}]}), + ({"lookup": {"one": {"displayName": "child", "future": True}}}, {"lookup": {"one": {"displayName": "child"}}}), + ({"choice": {"kind": "text", "text": "value", "future": True}}, {"choice": {"kind": "text", "text": "value"}}), + ({"choice": {"kind": "number", "number": 3, "future": True}}, {"choice": {"kind": "number", "number": 3}}), + ({"closed": {"label": "closed", "future": True}}, {"closed": {"label": "closed"}}), + ({"open": {"label": "open", "future": True}}, {"open": {"label": "open", "future": True}}), + ({"typed": {"label": "typed", "future": "value"}}, {"typed": {"label": "typed", "future": "value"}}), + ({"arbitrary": {"future": {"nullable": None}}}, {"arbitrary": {"future": {"nullable": None}}}), + ({"child": {"displayName": "child", "nullable": None}}, {"child": {"displayName": "child", "nullable": None}}), +] diff --git a/templates/templates/pythonv2/tests/primary/test_open_enum_union_discrimination.py b/templates/templates/pythonv2/tests/primary/test_open_enum_union_discrimination.py index 202c7ee36..405a2dbff 100644 --- a/templates/templates/pythonv2/tests/primary/test_open_enum_union_discrimination.py +++ b/templates/templates/pythonv2/tests/primary/test_open_enum_union_discrimination.py @@ -7,12 +7,14 @@ - Smart union mode correctly discriminates between union members based on field matches """ +import warnings + import pytest from typing import Union, Literal, Optional from pydantic import BaseModel, ValidationError, TypeAdapter, ConfigDict from .common_helpers import record_test -from openapi import SDK +from openapi import SDK, utils from openapi.models import shared, operations from openapi.types import UnrecognizedStr, UnrecognizedInt @@ -1368,3 +1370,28 @@ def test_unions_discriminated_open_enum(self): assert res.res is not None assert isinstance(res.res.json_, shared.ObjectWithOpenEnumStatus2) assert res.res.json_.reason == "User requested deactivation" + + +class TestOpenEnumScalarUnionSerialization: + @pytest.mark.parametrize("payload, wire", [ + ({"stringOrNumber": 1.5}, '{"stringOrNumber":1.5}'), + ({"stringOrNumber": "future"}, '{"stringOrNumber":"future"}'), + ({"stringOrNumber": "known"}, '{"stringOrNumber":"known"}'), + ({"stringOrBoolean": True}, '{"stringOrBoolean":true}'), + ({"stringOrBoolean": "future"}, '{"stringOrBoolean":"future"}'), + ({"integerOrNumber": 2.5}, '{"integerOrNumber":2.5}'), + ({"integerOrNumber": 1}, '{"integerOrNumber":1}'), + ]) + def test_scalar_member_serialization(self, payload, wire): + model = shared.OpenEnumScalarUnions.model_validate(payload) + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert utils.marshal_json(model, shared.OpenEnumScalarUnions) == wire + assert model.model_dump_json(by_alias=True, exclude_none=True) == wire + assert model.model_dump(by_alias=True, exclude_none=True) == payload + + def test_unrecognized_values_keep_their_type_in_python_mode(self): + model = shared.OpenEnumScalarUnions.model_validate({"stringOrNumber": "future", "stringOrBoolean": "future"}) + dumped = model.model_dump(by_alias=True, exclude_none=True) + assert type(dumped["stringOrNumber"]) is UnrecognizedStr + assert type(dumped["stringOrBoolean"]) is UnrecognizedStr diff --git a/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py new file mode 100644 index 000000000..a87a602a5 --- /dev/null +++ b/templates/templates/pythonv2/tests/primary/test_unknown_fields_additional.py @@ -0,0 +1,23 @@ +import json + +import pytest + +from openapi import SDK, utils +from openapi.models import shared +from .common_helpers import HTTPBIN_URL +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +@pytest.mark.parametrize("fields, known_fields", UNKNOWN_FIELD_CASES) +def test_unknown_fields_dropped_by_default(fields, known_fields): + model = shared.FieldEnvelopeInput.model_validate({"label": "root", **fields}) + expected = {"label": "root", **known_fields} + assert model.model_dump(mode="json", by_alias=True, exclude_none=True) == expected + assert json.loads(utils.marshal_json(model, shared.FieldEnvelopeInput)) == expected + + +def test_flattened_request_drops_nested_unknown_fields_by_default(): + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_flattened_unknown_fields(label="root", child={"displayName": "child", "future": True}) + assert response.http_meta.response.json()["json"] == {"label": "root", "child": {"displayName": "child"}} + assert response.object.model_dump(by_alias=True, exclude_none=True) == {"json": {"label": "root", "child": {"displayName": "child"}}} diff --git a/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py new file mode 100644 index 000000000..4420da343 --- /dev/null +++ b/templates/templates/pythonv2/tests/quaternary/test_unknown_fields_additional.py @@ -0,0 +1,123 @@ +import json +import warnings + +import pydantic +import pytest + +from openapi import SDK, AsyncSDK, utils +from openapi.models import shared +from openapi.types import UnrecognizedInt, UnrecognizedStr +from .common_helpers import HTTPBIN_URL, record_test +from .unknown_fields import drop_unexpected_fields +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +@pytest.mark.parametrize("child", [ + {"displayName": "child", "future": True}, + {"displayName": "child", "display_name": "unknown"}, + {"displayName": "child", "child": {"displayName": "grandchild", "future": {"nested": [None, 0]}}}, + {"displayName": "child", "nullable": None, "future": None}, +]) +def test_flattened_request_keeps_nested_unknown_fields(child): + record_test("unknown-fields-json-round-trip") + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_unknown_fields(label="root", child=child) + payload = {"label": "root", "child": child} + assert response.raw_response.json()["json"] == payload + assert response.object.json_.to_dict() == payload + assert json.loads(response.object.json_.to_json()) == payload + + +@pytest.mark.asyncio() +async def test_flattened_request_keeps_nested_unknown_fields_async(): + child = {"displayName": "child", "child": {"displayName": "grandchild", "future": {"nested": [None, 0]}}} + async with AsyncSDK(server_url=HTTPBIN_URL) as sdk: + response = await sdk.echo_unknown_fields(label="root", child=child) + payload = {"label": "root", "child": child} + assert response.raw_response.json()["json"] == payload + assert response.object.json_.to_dict() == payload + assert json.loads(response.object.json_.to_json()) == payload + + +def test_flattened_request_keeps_unknown_fields_on_nested_models(): + child = shared.FieldNode(display_name="child", future=None) + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_flattened_unknown_fields(label="root", child=child) + assert response.raw_response.json()["json"] == {"label": "root", "child": {"displayName": "child", "future": None}} + + +def test_flattened_request_has_no_argument_for_top_level_unknown_fields(): + with SDK(server_url=HTTPBIN_URL) as sdk: + with pytest.raises(TypeError): + sdk.echo_flattened_unknown_fields(label="root", future=True) # type: ignore[call-arg] + + +@pytest.mark.parametrize("fields", [fields for fields, _ in UNKNOWN_FIELD_CASES]) +def test_to_dict_round_trips_unknown_fields(fields): + payload = {"label": "root", **fields} + model = shared.FieldEnvelopeInput.model_validate(payload) + assert model.to_dict(mode="json") == payload + assert json.loads(model.to_json()) == payload + assert shared.FieldEnvelopeInput.model_validate(model.to_dict()).to_dict(mode="json") == payload + + +@pytest.mark.parametrize("options, expected", [ + ({}, {"displayName": "child", "nullable": None, "future": None}), + ({"use_api_names": False}, {"display_name": "child", "nullable": None, "future": None}), + ({"exclude_unset": False}, {"displayName": "child", "nullable": None, "future": None}), + ({"exclude_defaults": True}, {"displayName": "child", "nullable": None, "future": None}), + ({"exclude_none": True}, {"displayName": "child"}), +]) +def test_model_serialization_methods(options, expected): + model = shared.FieldNode.model_validate({"displayName": "child", "nullable": None, "future": None}) + assert model.to_dict(**options) == expected + assert model.to_dict(mode="json", **options) == expected + assert json.loads(model.to_json(**options)) == expected + + +@pytest.mark.parametrize("options", [{"exclude_none": True}, {"exclude_defaults": True}]) +def test_excluded_field_keeps_unknown_field_named_like_it(options): + model = shared.FieldEnvelopeInput.model_validate({"label": "root", "plainChoice": None, "plain_choice": "unknown"}) + assert model.to_dict(**options) == {"label": "root", "plain_choice": "unknown"} + assert model.model_dump(by_alias=True, **options) == {"label": "root", "plain_choice": "unknown"} + + +def test_drop_unexpected_fields_ignores_only_undeclared_fields(): + expected = shared.TypedFields(label="typed") + assert drop_unexpected_fields(shared.TypedFields.model_validate({"label": "typed", "future": "value"}), expected) != expected + assert drop_unexpected_fields(shared.FieldNode.model_validate({"displayName": "child", "future": True}), shared.FieldNode(display_name="child")) == shared.FieldNode(display_name="child") + + +def test_model_serialization_methods_field_names(): + model = shared.FieldEnvelopeInput.model_validate({"label": "root", "child": {"displayName": "child", "child": {"displayName": "grandchild"}}, "futureKind": "third", "plainChoice": {"text": "t"}, "future": True}) + api_names = {"label": "root", "child": {"displayName": "child", "child": {"displayName": "grandchild"}}, "futureKind": "third", "plainChoice": {"text": "t"}, "future": True} + field_names = {"label": "root", "child": {"display_name": "child", "child": {"display_name": "grandchild"}}, "future_kind": "third", "plain_choice": {"text": "t"}, "future": True} + assert model.model_dump(exclude_unset=True) == api_names + assert model.to_dict() == api_names + assert model.to_dict(use_api_names=False) == field_names + assert json.loads(model.to_json(use_api_names=False)) == field_names + assert shared.FieldEnvelopeInput.model_validate(model.to_dict(use_api_names=False)).to_dict() == api_names + + +@pytest.mark.parametrize("typ, value", [(UnrecognizedStr, "future"), (UnrecognizedInt, 3)]) +def test_unrecognized_enum_serialization(typ, value): + adapter = pydantic.TypeAdapter(typ) + enum_value = adapter.validate_python(value) + assert isinstance(enum_value, typ) + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert adapter.dump_python(enum_value) is enum_value + assert adapter.dump_python(enum_value, mode="json") == value + assert json.loads(adapter.dump_json(enum_value)) == value + with pytest.raises(pydantic.ValidationError): + adapter.validate_python(value, strict=True) + + +def test_unrecognized_enum_generated_model(): + model = shared.FieldEnvelope.model_validate({"label": "root", "futureKind": "third", "futureCode": 3}) + with warnings.catch_warnings(): + warnings.simplefilter("error") + dumped = model.model_dump(by_alias=True) + assert type(dumped["futureKind"]) is UnrecognizedStr + assert type(dumped["futureCode"]) is UnrecognizedInt + assert json.loads(utils.marshal_json(model, shared.FieldEnvelope)) == {"label": "root", "futureKind": "third", "futureCode": 3} diff --git a/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py new file mode 100644 index 000000000..6aa3af1e5 --- /dev/null +++ b/templates/templates/pythonv2/tests/secondary/test_unknown_fields_additional.py @@ -0,0 +1,122 @@ +import json +from typing import Annotated, Any, List, Optional + +import httpx +import pydantic +import pytest + +from openapi import SDK, utils +from openapi.models import errors, shared +from openapi.types import UNSET, BaseModel, Nullable +from .common_helpers import HTTPBIN_URL, record_test +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +CASES = [fields for fields, _ in UNKNOWN_FIELD_CASES] + + +def echo(request): + sent = [] + with httpx.Client(event_hooks={"request": [sent.append]}) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + response = sdk.echo_unknown_fields(request=request) + return response, json.loads(sent[-1].content) + + +@pytest.mark.parametrize("fields", CASES) +@pytest.mark.parametrize("as_model", [False, True]) +def test_unknown_fields_echo(fields, as_model): + record_test("unknown-fields-json-round-trip") + payload = {"label": "root", **fields} + request = shared.FieldEnvelope.model_validate(payload) if as_model else payload + response, sent = echo(request) + assert sent == payload + assert response.json_.model_dump(mode="json", by_alias=True) == payload + assert json.loads(response.json_.model_dump_json(by_alias=True)) == payload + assert "headers" in response.model_dump() + + +def test_response_only_unknown_fields(): + payload = {"json": {"label": "root", "child": {"displayName": "child", "future": True}}, "future": {"value": 1}} + transport = httpx.MockTransport(lambda request: httpx.Response(200, json=payload)) + with httpx.Client(transport=transport) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + response = sdk.echo_unknown_fields(request={"label": "root"}) + dumped = response.model_dump(mode="json", by_alias=True) + assert "future" in dumped + assert "future" in dumped["json"]["child"] + + +def test_response_model_can_be_sent_again(): + payload = {"label": "root", "child": {"displayName": "child", "future": {"enabled": False}}} + first, _ = echo(payload) + second, sent = echo(first.json_.model_dump(by_alias=True)) + assert sent == payload + assert second.json_.model_dump(by_alias=True, exclude_none=True) == payload + + +def test_error_response_unknown_fields(): + payload = {"message": "invalid", "future": None, "detail": {"displayName": "child", "future": {"flag": True}}} + transport = httpx.MockTransport(lambda request: httpx.Response(400, json=payload)) + with httpx.Client(transport=transport) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + with pytest.raises(errors.EchoUnknownFieldsResponseBody) as caught: + sdk.echo_unknown_fields(request={"label": "root"}) + dumped = getattr(caught.value, "data").model_dump(mode="json", by_alias=True) + assert "future" in dumped + assert "future" in dumped["detail"] + + +def test_omitted_json_query_stays_empty(): + sent = [] + with httpx.Client(event_hooks={"request": [sent.append]}) as client: + with SDK(server_url=HTTPBIN_URL, client=client) as sdk: + sdk.echo_optional_json_query() + assert sent[-1].url.params["filter"] == "" + + +def test_typed_additional_properties_still_validate(): + with pytest.raises(pydantic.ValidationError): + shared.FieldEnvelope.model_validate({"label": "root", "typed": {"future": {"invalid": True}}}) + + +def test_unknown_fields_dump_filters(): + model = shared.FieldEnvelope.model_validate({"label": "root", "future": True, "future_null": None}) + assert "future_null" in model.model_dump() + assert "future_null" not in model.model_dump(exclude_none=True) + assert "future" not in model.model_dump(exclude={"future"}) + assert model.model_dump(include={"label"}) == {"label": "root"} + + +def test_base_model_unknown_field_policy(): + class CustomModel(BaseModel): + label: str + + model = CustomModel(label="value", future=None) + assert "future" in model.model_dump() + + +def test_invalid_nullable_value_becomes_unset_like_default_policy(): + node = shared.FieldNode.model_validate({"displayName": "child", "nullable": {"future": True}}) + assert node.nullable == UNSET + + +def test_helper_names_not_reserved_without_methods(): + assert set(shared.MethodNames.model_fields) == {"to_dict", "to_json"} + assert not hasattr(shared.FieldEnvelope, "to_dict") + + +@pytest.mark.parametrize("typ", [Optional[int], Optional[List[int]], Optional[shared.FieldNode], Any]) +def test_top_level_none_serialization(typ): + assert utils.marshal_json(None, typ) == "" + + +def test_top_level_none_still_validates(): + with pytest.raises(pydantic.ValidationError): + utils.marshal_json(None, shared.FieldNode) + assert utils.marshal_json(None, Nullable[shared.FieldNode]) == "null" + + +def test_coerced_top_level_none_stays_empty(): + typ = Annotated[Optional[int], pydantic.BeforeValidator(lambda value: None)] + assert utils.marshal_json("empty", typ) == "" diff --git a/templates/templates/pythonv2/tests/tertiary/test_unknown_fields_additional.py b/templates/templates/pythonv2/tests/tertiary/test_unknown_fields_additional.py new file mode 100644 index 000000000..4f0bf4abf --- /dev/null +++ b/templates/templates/pythonv2/tests/tertiary/test_unknown_fields_additional.py @@ -0,0 +1,99 @@ +import json + +import pytest + +from openapi import SDK, utils +from openapi.models import shared +from .common_helpers import HTTPBIN_URL +from .unknown_fields_cases import UNKNOWN_FIELD_CASES + + +@pytest.mark.parametrize("fields, known_fields", UNKNOWN_FIELD_CASES) +def test_to_dict_drops_unknown_fields(fields, known_fields): + model = shared.FieldEnvelopeInput.model_validate({"label": "root", **fields}) + expected = {"label": "root", **known_fields} + assert model.to_dict(mode="json") == expected + assert json.loads(model.to_json()) == expected + assert shared.FieldEnvelopeInput.model_validate(model.to_dict()).to_dict(mode="json") == expected + + +def test_flattened_request_drops_nested_unknown_fields(): + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.echo_flattened_unknown_fields(label="root", child={"displayName": "child", "future": True}) + assert response.raw_response.json()["json"] == {"label": "root", "child": {"displayName": "child"}} + assert response.object.json_.to_dict() == {"label": "root", "child": {"displayName": "child"}} + assert "headers" not in response.object.to_dict() + + +@pytest.mark.parametrize("options, expected", [ + ({}, {"displayName": "child", "nullable": None}), + ({"use_api_names": False}, {"display_name": "child", "nullable": None}), + ({"exclude_unset": False}, {"displayName": "child", "nullable": None}), + ({"exclude_defaults": True}, {"displayName": "child", "nullable": None}), + ({"exclude_none": True}, {"displayName": "child"}), +]) +def test_model_serialization_methods(options, expected): + model = shared.FieldNode.model_validate({"displayName": "child", "nullable": None}) + assert model.to_dict(**options) == expected + assert model.to_dict(mode="json", **options) == expected + assert json.loads(model.to_json(**options)) == expected + + +def test_exclude_none_keeps_explicit_nulls_on_the_wire(): + model = shared.FieldNode(display_name="child", nullable=None) + assert model.to_dict(exclude_none=True) == {"displayName": "child"} + assert json.loads(utils.marshal_json(model, shared.FieldNode)) == {"displayName": "child", "nullable": None} + + +def test_excluded_fields_stay_out_of_serialization_methods(): + model = shared.DefaultsAndConsts(normal_field="x") + assert "defaultStrNullable" not in model.to_dict() + assert model.to_dict(exclude_unset=False)["defaultStrNullable"] is None + assert shared.DefaultsAndConsts(normal_field="x", default_str_nullable=None).to_dict()["defaultStrNullable"] is None + assert json.loads(utils.marshal_json(model, shared.DefaultsAndConsts))["defaultStrNullable"] is None + node = shared.FieldNode(display_name="child", nullable=None) + assert node.model_dump(by_alias=True, exclude={"display_name"}) == {"nullable": None} + assert node.model_dump(by_alias=True, exclude_none=True) == {"displayName": "child", "nullable": None} + + +def test_model_serialization_methods_field_names(): + model = shared.FieldEnvelopeInput.model_validate({"label": "root", "child": {"displayName": "child"}, "futureKind": "third", "plainChoice": {"text": "t"}}) + api_names = {"label": "root", "child": {"displayName": "child"}, "futureKind": "third", "plainChoice": {"text": "t"}} + field_names = {"label": "root", "child": {"display_name": "child"}, "future_kind": "third", "plain_choice": {"text": "t"}} + assert model.to_dict() == api_names + assert model.to_dict(use_api_names=False) == field_names + assert json.loads(model.to_json(use_api_names=False)) == field_names + assert shared.FieldEnvelopeInput.model_validate(model.to_dict(use_api_names=False)).to_dict() == api_names + + +def test_helper_names_reserved_only_on_models(): + assert set(shared.MethodNames.model_fields) == {"to_dict_", "to_json_"} + model = shared.MethodNames.model_validate({"toDict": "d", "toJson": "j"}) + assert model.to_dict() == {"toDict": "d", "toJson": "j"} + with SDK(server_url=HTTPBIN_URL) as sdk: + response = sdk.to_json.echo_grouped_fields() + assert response.status_code == 200 + + +def test_model_serialization_methods_keep_consts(): + model = shared.FieldEnvelopeInput(label="root", choice=shared.TextChoice(text="t"), constant=shared.ConstFields()) + expected = {"label": "root", "choice": {"kind": "text", "text": "t"}, "constant": {"kind": "fixed"}} + wire = {"label": "root", "choice": {"kind": "text", "text": "t"}, "constant": {"kind": "fixed", "priority": 1}} + assert json.loads(utils.marshal_json(model, shared.FieldEnvelopeInput)) == wire + assert model.to_dict() == expected + assert model.to_dict(exclude_unset=False) == wire + assert json.loads(model.to_json()) == expected + assert shared.FieldEnvelopeInput.model_validate(model.to_dict()).to_dict() == expected + assert shared.FieldEnvelopeInput.model_validate(wire).to_dict() == wire + assert shared.ConstFields(priority=1).to_dict() == {"kind": "fixed", "priority": 1} + assert shared.ConstFields(priority=1).to_dict(exclude_defaults=True) == {} + assert shared.ConstFields(priority=2).to_dict(exclude_defaults=True) == {"priority": 2} + + +def test_optional_null_consts_keep_wire_output(): + model = shared.ObjectWithNullEnums() + assert json.loads(utils.marshal_json(model, shared.ObjectWithNullEnums)) == {"nullEnum": None} + assert model.to_dict() == {"nullEnum": None} + explicit = shared.ObjectWithNullEnums(opt_null_enum=None) + assert json.loads(utils.marshal_json(explicit, shared.ObjectWithNullEnums)) == {"nullEnum": None, "optNullEnum": None} + assert explicit.to_dict() == {"nullEnum": None, "optNullEnum": None} diff --git a/templates/templates/ruby/features.ts b/templates/templates/ruby/features.ts index 3435fe0de..fec3e49fc 100644 --- a/templates/templates/ruby/features.ts +++ b/templates/templates/ruby/features.ts @@ -475,6 +475,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/typescriptv2/features.ts b/templates/templates/typescriptv2/features.ts index b052b86a0..4f9056742 100644 --- a/templates/templates/typescriptv2/features.ts +++ b/templates/templates/typescriptv2/features.ts @@ -423,6 +423,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/templates/templates/unity/features.ts b/templates/templates/unity/features.ts index e69b828ac..4276f2af6 100644 --- a/templates/templates/unity/features.ts +++ b/templates/templates/unity/features.ts @@ -596,6 +596,7 @@ function isTestSkipped(test: string): boolean { "event-stream-absent-required-union", "event-stream-equivalent-number-lexemes", "event-stream-null-and-undeclared-fields", + "unknown-fields-json-round-trip", ].includes(test); } diff --git a/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml b/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml index a61edf83b..b8c4abd36 100644 --- a/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/quaternary/pythonv2/.speakeasy/gen.yaml @@ -35,6 +35,8 @@ python: responseFormat: envelope responseSchemaValidation: lenient flattenRequests: true + preserveUnknownFields: true + modelSerializationMethods: true pytestFilterWarnings: - error::RuntimeWarning # Ensure __del__ method warnings are raised diff --git a/tests/config/review/pythonv2/.speakeasy/gen.yaml b/tests/config/review/pythonv2/.speakeasy/gen.yaml index 7d4876daa..b06dd1876 100644 --- a/tests/config/review/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/review/pythonv2/.speakeasy/gen.yaml @@ -88,6 +88,7 @@ python: methodArguments: infer-optional-args methodTimeoutArgument: timeout-ms methodTimeoutUnits: milliseconds + modelSerializationMethods: false moduleName: speakeasy.new_openapi multipartArrayFormat: standard optionalDependencies: @@ -99,6 +100,7 @@ python: packageManager: uv packageName: openapi preApplyUnionDiscriminators: false + preserveUnknownFields: false pytestFilterWarnings: [] pytestTimeout: 0 rawResponseHelpers: false diff --git a/tests/config/secondary/pythonv2/.speakeasy/gen.yaml b/tests/config/secondary/pythonv2/.speakeasy/gen.yaml index 97ddd5bd2..fa854c67a 100644 --- a/tests/config/secondary/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/secondary/pythonv2/.speakeasy/gen.yaml @@ -38,6 +38,7 @@ python: webhooks: models/webhooks maxMethodParams: 5 flattenRequests: false + preserveUnknownFields: true packageName: openapi rawResponseHelpers: true responseFormat: flat diff --git a/tests/config/tertiary/pythonv2/.speakeasy/gen.yaml b/tests/config/tertiary/pythonv2/.speakeasy/gen.yaml index ffe89c4a4..41aa32644 100644 --- a/tests/config/tertiary/pythonv2/.speakeasy/gen.yaml +++ b/tests/config/tertiary/pythonv2/.speakeasy/gen.yaml @@ -32,6 +32,7 @@ python: relative: true maxMethodParams: 0 flattenRequests: true + modelSerializationMethods: true packageName: openapi responseFormat: envelope inputTypedDictSuffix: Param diff --git a/tests/overlays/tertiary/typescriptv2/overlay.yaml b/tests/overlays/tertiary/typescriptv2/overlay.yaml new file mode 100644 index 000000000..fcc524d36 --- /dev/null +++ b/tests/overlays/tertiary/typescriptv2/overlay.yaml @@ -0,0 +1,9 @@ +overlay: 1.0.0 +info: + title: Overlay ./tests/specs/uber.yaml => tertiary typescriptv2 test spec + version: 0.0.0 +actions: + # Inline success and error bodies share a generated name, which collides on import + # without index modules, flat responses and the legacy naming scheme + - target: $.paths.*.*[?(@.operationId == 'echoUnknownFields')] + remove: true diff --git a/tests/specs/fragments/primary/open-enum-scalar-unions.yaml b/tests/specs/fragments/primary/open-enum-scalar-unions.yaml new file mode 100644 index 000000000..15a8e39db --- /dev/null +++ b/tests/specs/fragments/primary/open-enum-scalar-unions.yaml @@ -0,0 +1,40 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +components: + schemas: + OpenEnumScalarUnions: + x-speakeasy-include: true + type: object + description: >- + Properties whose union combines an open enum with a number or boolean. + A number or boolean held in such a union must keep its value and JSON + type when serialized, rather than being converted as if it were an + unrecognized enum value. + properties: + stringOrNumber: + anyOf: + - $ref: '#/components/schemas/OpenEnumScalarString' + - type: number + stringOrBoolean: + anyOf: + - $ref: '#/components/schemas/OpenEnumScalarString' + - type: boolean + integerOrNumber: + anyOf: + - $ref: '#/components/schemas/OpenEnumScalarInteger' + - type: number + OpenEnumScalarString: + type: string + enum: [known] + x-speakeasy-unknown-values: allow + OpenEnumScalarInteger: + type: integer + enum: [1, 2] + x-speakeasy-unknown-values: allow diff --git a/tests/specs/fragments/uber/unknown-fields.yaml b/tests/specs/fragments/uber/unknown-fields.yaml new file mode 100644 index 000000000..b7f39f586 --- /dev/null +++ b/tests/specs/fragments/uber/unknown-fields.yaml @@ -0,0 +1,284 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +paths: + /anything/unknown-fields-group: + get: + operationId: echoGroupedFields + x-speakeasy-group: toJson + x-speakeasy-exclude-test-sdk: [ruby] + description: Belongs to a group named like a common model serialization method, so the group accessor must keep its name when models reserve that method name. + responses: + '200': + description: The echoed request URL. + content: + application/json: + schema: + type: object + properties: + url: + type: string + /anything/unknown-fields-query: + get: + operationId: echoOptionalJsonQuery + description: Echoes an optional JSON query parameter without changing its omission semantics. + parameters: + - name: filter + in: query + required: false + content: + application/json: + schema: + title: UnknownFieldsQueryFilter + type: object + properties: + label: + type: string + responses: + '200': + description: The echoed request URL. + content: + application/json: + schema: + type: object + properties: + url: + type: string + /anything/unknown-fields: + post: + operationId: echoUnknownFields + x-speakeasy-exclude-test-sdk: [javav2] + description: Echoes nested objects to exercise preservation of undeclared fields in requests and responses. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FieldEnvelope' + responses: + '200': + description: The echoed JSON request and transport metadata. + content: + application/json: + schema: + type: object + required: [json] + properties: + json: + $ref: '#/components/schemas/FieldEnvelope' + '400': + description: A structured error with additional server fields. + content: + application/json: + schema: + type: object + required: [message] + properties: + message: + type: string + detail: + $ref: '#/components/schemas/FieldNode' + /anything/unknown-fields-flattened: + post: + operationId: echoFlattenedUnknownFields + description: Echoes a body small enough to be flattened into operation arguments, so undeclared fields can only travel in nested objects. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/FlattenedFields' + responses: + '200': + description: The echoed JSON request. + content: + application/json: + schema: + type: object + required: [json] + properties: + json: + $ref: '#/components/schemas/FlattenedFields' +components: + schemas: + FieldEnvelope: + type: object + required: [label] + properties: + label: + type: string + child: + $ref: '#/components/schemas/FieldNode' + children: + type: array + items: + $ref: '#/components/schemas/FieldNode' + lookup: + type: object + additionalProperties: + $ref: '#/components/schemas/FieldNode' + choice: + oneOf: + - $ref: '#/components/schemas/TextChoice' + - $ref: '#/components/schemas/NumberChoice' + discriminator: + propertyName: kind + mapping: + text: '#/components/schemas/TextChoice' + number: '#/components/schemas/NumberChoice' + plainChoice: + anyOf: + - $ref: '#/components/schemas/TextFields' + - $ref: '#/components/schemas/NumberFields' + requiredOnly: + $ref: '#/components/schemas/RequiredFields' + composed: + allOf: + - $ref: '#/components/schemas/FieldNode' + - type: object + properties: + category: + type: string + split: + $ref: '#/components/schemas/SplitFields' + empty: + $ref: '#/components/schemas/EmptyFields' + open: + $ref: '#/components/schemas/OpenFields' + typed: + $ref: '#/components/schemas/TypedFields' + closed: + $ref: '#/components/schemas/ClosedFields' + constant: + $ref: '#/components/schemas/ConstFields' + arbitrary: {} + futureKind: + type: string + enum: [first, second] + x-speakeasy-unknown-values: allow + futureCode: + type: integer + enum: [1, 2] + x-speakeasy-unknown-values: allow + methodNames: + type: object + properties: + toDict: + type: string + toJson: + type: string + FieldNode: + type: object + required: [displayName] + properties: + displayName: + type: string + nullable: + type: [string, 'null'] + child: + $ref: '#/components/schemas/FieldNode' + TextChoice: + type: object + required: [kind, text] + properties: + kind: + $ref: '#/components/schemas/TextChoiceKind' + text: + type: string + NumberChoice: + type: object + required: [kind, number] + properties: + kind: + $ref: '#/components/schemas/NumberChoiceKind' + number: + type: integer + TextChoiceKind: + type: string + enum: [text] + NumberChoiceKind: + type: string + enum: [number] + OpenFields: + type: object + additionalProperties: true + properties: + label: + type: string + TypedFields: + type: object + additionalProperties: + type: string + properties: + label: + type: string + ClosedFields: + type: object + additionalProperties: false + properties: + label: + type: string + TextFields: + type: object + required: [text] + properties: + text: + type: string + label: + type: string + NumberFields: + type: object + required: [number] + properties: + number: + type: integer + label: + type: string + RequiredFields: + type: object + required: [label] + properties: + label: + type: string + SplitFields: + type: object + properties: + label: + type: string + id: + type: string + readOnly: true + secret: + type: string + writeOnly: true + EmptyFields: + type: object + additionalProperties: false + ConstFields: + type: object + description: A required const alongside optional fields, one of them with a default, so values are populated by their defaults rather than by the caller. + required: [kind] + properties: + kind: + type: string + const: fixed + label: + type: string + priority: + type: integer + default: 1 + FlattenedFields: + type: object + description: A body with few enough properties to be flattened into operation arguments. + required: [label] + properties: + label: + type: string + child: + $ref: '#/components/schemas/FieldNode' diff --git a/zSDKs/sdk-pythonv2/.speakeasy/gen.lock b/zSDKs/sdk-pythonv2/.speakeasy/gen.lock index e672ea8e3..87edb846a 100644 --- a/zSDKs/sdk-pythonv2/.speakeasy/gen.lock +++ b/zSDKs/sdk-pythonv2/.speakeasy/gen.lock @@ -6,7 +6,7 @@ management: speakeasyVersion: internal generationVersion: internal releaseVersion: 0.0.1 - configChecksum: d67daf2ec44e985625b98d32573578c6 + configChecksum: 3b5207abcf7df0a028f020866f6e1112 repoURL: https://github.com/speakeasy-sdks/test-sdk installationURL: https://github.com/speakeasy-sdks/test-sdk.git features: @@ -746,7 +746,7 @@ trackedFiles: src/speakeasy/new_openapi/types/base64fileinput.py: last_write_checksum: sha1:1522687ae3398374c35710cad993a6e82b5ab99d src/speakeasy/new_openapi/types/basemodel.py: - last_write_checksum: sha1:10d84aedeb9d35edfdadf2c3020caa1d24d8b584 + last_write_checksum: sha1:4995f30bbd8f292e6a1608ee4bf5a730f0a5d14e src/speakeasy/new_openapi/types_.py: last_write_checksum: sha1:05e36f6428f04b1ea7d2609aa55f3966868b58f0 src/speakeasy/new_openapi/utils/__init__.py: diff --git a/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml b/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml index 7d4876daa..b06dd1876 100644 --- a/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml +++ b/zSDKs/sdk-pythonv2/.speakeasy/gen.yaml @@ -88,6 +88,7 @@ python: methodArguments: infer-optional-args methodTimeoutArgument: timeout-ms methodTimeoutUnits: milliseconds + modelSerializationMethods: false moduleName: speakeasy.new_openapi multipartArrayFormat: standard optionalDependencies: @@ -99,6 +100,7 @@ python: packageManager: uv packageName: openapi preApplyUnionDiscriminators: false + preserveUnknownFields: false pytestFilterWarnings: [] pytestTimeout: 0 rawResponseHelpers: false diff --git a/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py b/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py index a9a640a1a..e7c6fe749 100644 --- a/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py +++ b/zSDKs/sdk-pythonv2/src/speakeasy/new_openapi/types/basemodel.py @@ -2,8 +2,8 @@ from pydantic import ConfigDict, model_serializer from pydantic import BaseModel as PydanticBaseModel -from pydantic_core import core_schema -from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union +from pydantic_core import PydanticSerializationUnexpectedValue, core_schema +from typing import TYPE_CHECKING, Any, Literal, Optional, Type, TypeVar, Union from typing_extensions import TypeAliasType, TypeAlias @@ -37,12 +37,28 @@ def __bool__(self) -> Literal[False]: ) +def _unrecognized_serialization( + cls: Type[Union[str, int]], +) -> core_schema.PlainSerializerFunctionSerSchema: + def serialize(value: Any, info: core_schema.SerializationInfo) -> Any: + # Decline other values so union serialization tries the remaining members + if not isinstance(value, cls): + raise PydanticSerializationUnexpectedValue(f"Expected {cls.__name__}") + if info.mode == "python": + return value + return str(value) if isinstance(value, str) else int(value) + + return core_schema.plain_serializer_function_ser_schema(serialize, info_arg=True) + + class UnrecognizedStr(str): @classmethod - def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> core_schema.CoreSchema: + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: Any + ) -> core_schema.CoreSchema: # Make UnrecognizedStr only work in lax mode, not strict mode # This makes it a "fallback" option when more specific types (like Literals) don't match - def validate_lax(v: Any) -> 'UnrecognizedStr': + def validate_lax(v: Any) -> "UnrecognizedStr": if isinstance(v, cls): return v return cls(str(v)) @@ -51,27 +67,36 @@ def validate_lax(v: Any) -> 'UnrecognizedStr': # This forces Pydantic to prefer other union members in strict mode # and only fall back to UnrecognizedStr in lax mode return core_schema.lax_or_strict_schema( - lax_schema=core_schema.chain_schema([ - core_schema.str_schema(), - core_schema.no_info_plain_validator_function(validate_lax) - ]), + lax_schema=core_schema.chain_schema( + [ + core_schema.str_schema(), + core_schema.no_info_plain_validator_function(validate_lax), + ] + ), strict_schema=core_schema.none_schema(), # Always fails in strict mode + serialization=_unrecognized_serialization(cls), ) class UnrecognizedInt(int): @classmethod - def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> core_schema.CoreSchema: + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: Any + ) -> core_schema.CoreSchema: # Make UnrecognizedInt only work in lax mode, not strict mode # This makes it a "fallback" option when more specific types (like Literals) don't match - def validate_lax(v: Any) -> 'UnrecognizedInt': + def validate_lax(v: Any) -> "UnrecognizedInt": if isinstance(v, cls): return v return cls(int(v)) + return core_schema.lax_or_strict_schema( - lax_schema=core_schema.chain_schema([ - core_schema.int_schema(), - core_schema.no_info_plain_validator_function(validate_lax) - ]), + lax_schema=core_schema.chain_schema( + [ + core_schema.int_schema(), + core_schema.no_info_plain_validator_function(validate_lax), + ] + ), strict_schema=core_schema.none_schema(), # Always fails in strict mode + serialization=_unrecognized_serialization(cls), )