Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ linters:
- gomodguard
- gomodguard_v2
- exhaustruct
- exhaustruct_v5
- nlreturn
- nonamedreturns
- noinlineerr
Expand Down
53 changes: 53 additions & 0 deletions analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,15 @@ type referenceAnalysis struct {
parameterItems map[string]spec.Ref
allRefs map[string]spec.Ref
pathItems map[string]spec.Ref

// unmappedRefs holds the $ref found under keywords the Swagger 2.0 model does not map, which
// land in [spec.Schema.ExtraProps] as raw JSON: propertyNames, contains, if/then/else, $defs.
//
// They are kept apart from allRefs and schemas on purpose. Flatten needs them to import their
// target and rewrite the pointer; every other consumer of this analysis - AllRefs,
// AllReferences, AllDefinitionReferences and what go-swagger builds on them - addresses schemas
// through the model, and a key naming a raw JSON node is of no use there.
unmappedRefs map[string]spec.Ref
}

func (r *referenceAnalysis) addRef(key string, ref spec.Ref) {
Expand All @@ -48,6 +57,11 @@ func (r *referenceAnalysis) addItemsRef(key string, items *spec.Items, location
}
}

// addUnmappedRef records a $ref held by a keyword the model does not map.
func (r *referenceAnalysis) addUnmappedRef(key string, ref spec.Ref) {
r.unmappedRefs["#"+key] = ref
}

func (r *referenceAnalysis) addSchemaRef(key string, ref SchemaRef) {
r.schemas["#"+key] = ref.Schema.Ref
r.addRef(key, ref.Schema.Ref)
Expand Down Expand Up @@ -739,6 +753,7 @@ func (s *Spec) reset() {
s.references.headerItems = make(map[string]spec.Ref, allocLargeMap)
s.references.parameterItems = make(map[string]spec.Ref, allocLargeMap)
s.references.allRefs = make(map[string]spec.Ref, allocLargeMap)
s.references.unmappedRefs = make(map[string]spec.Ref, allocSmallMap)
s.patterns.parameters = make(map[string]string, allocLargeMap)
s.patterns.headers = make(map[string]string, allocLargeMap)
s.patterns.items = make(map[string]string, allocLargeMap)
Expand Down Expand Up @@ -972,6 +987,42 @@ func (s *Spec) analyzeResponse(prefix string, k int, res spec.Response) {
}
}

// analyzeUnmapped records the $ref held by the keywords of a schema that the Swagger 2.0 model
// does not map, which json.Unmarshal leaves in ExtraProps as raw JSON.
//
// The keys it produces address the node holding the $ref, so "#/definitions/deep/propertyNames"
// or "#/definitions/deep/if/anyOf/0". [replace.UpdateRef] writes to them through the same
// jsonpointer call every other key goes through.
func (s *Spec) analyzeUnmapped(prefix string, extra map[string]any) {
for key := range extra {
s.analyzeUnmappedNode(slashpath.Join(prefix, jsonpointer.Escape(key)), extra[key])
}
}

func (s *Spec) analyzeUnmappedNode(refURI string, node any) {
switch value := node.(type) {
case map[string]any:
if raw, ok := value["$ref"].(string); ok {
ref, err := spec.NewRef(raw)
if err != nil {
return // a string under a "$ref" key is not necessarily a reference
}

s.references.addUnmappedRef(refURI, ref)

return // a $ref makes its siblings irrelevant
}

for key := range value {
s.analyzeUnmappedNode(slashpath.Join(refURI, jsonpointer.Escape(key)), value[key])
}
case []any:
for i := range value {
s.analyzeUnmappedNode(slashpath.Join(refURI, strconv.Itoa(i)), value[i])
}
}
}

func (s *Spec) analyzeSchema(name string, schema *spec.Schema, prefix string) {
refURI := slashpath.Join(prefix, jsonpointer.Escape(name))
schRef := SchemaRef{
Expand All @@ -995,6 +1046,8 @@ func (s *Spec) analyzeSchema(name string, schema *spec.Schema, prefix string) {
s.enums.addSchemaEnum(refURI, schema.Enum)
}

s.analyzeUnmapped(refURI, schema.ExtraProps)

for k, v := range schema.Definitions {
s.analyzeSchema(k, &v, slashpath.Join(refURI, "definitions"))
}
Expand Down
47 changes: 33 additions & 14 deletions flatten.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package analysis

import (
"log"
"maps"
"path"
"slices"
"sort"
Expand Down Expand Up @@ -179,18 +180,20 @@ func normalizeRef(opts *FlattenOpts) error {
debugLog("normalizeRef")

altered := false
for k, w := range opts.Spec.references.allRefs {
if !strings.HasPrefix(w.String(), opts.BasePath+definitionsPath) { // may be a mix of / and \, depending on OS
continue
}
for _, refs := range []map[string]spec.Ref{opts.Spec.references.allRefs, opts.Spec.references.unmappedRefs} {
for k, w := range refs {
if !strings.HasPrefix(w.String(), opts.BasePath+definitionsPath) { // may be a mix of / and \, depending on OS
continue
}

altered = true
debugLog("stripping absolute path for: %s", w.String())
altered = true
debugLog("stripping absolute path for: %s", w.String())

// strip the base path from definition
if err := replace.UpdateRef(opts.Swagger(), k,
spec.MustCreateRef(path.Join(definitionsPath, path.Base(w.String())))); err != nil {
return err
// strip the base path from definition
if err := replace.UpdateRef(opts.Swagger(), k,
spec.MustCreateRef(path.Join(definitionsPath, path.Base(w.String())))); err != nil {
return err
}
}
}

Expand Down Expand Up @@ -276,6 +279,10 @@ func removeUnusedSinglePass(opts *FlattenOpts) (hasRemoved bool) {
delete(expected, k)
}

for _, ref := range opts.Spec.references.unmappedRefs {
delete(expected, ref.String())
}

for k := range expected {
hasRemoved = true
debugLog("removing unused definition %s", path.Base(k))
Expand Down Expand Up @@ -327,9 +334,11 @@ func importNewRef(entry sortref.RefRevIdx, refStr string, opts *FlattenOpts) err
partialAnalyzer.analyzeSchema("", sch, "/")

// now rewrite those refs with rebase
for key, ref := range partialAnalyzer.references.allRefs {
if err := replace.UpdateRef(sch, key, spec.MustCreateRef(normalize.RebaseRef(entry.Ref.String(), ref.String()))); err != nil {
return ErrRewriteRef(key, entry.Ref.String(), err)
for _, refs := range []map[string]spec.Ref{partialAnalyzer.references.allRefs, partialAnalyzer.references.unmappedRefs} {
for key, ref := range refs {
if err := replace.UpdateRef(sch, key, spec.MustCreateRef(normalize.RebaseRef(entry.Ref.String(), ref.String()))); err != nil {
return ErrRewriteRef(key, entry.Ref.String(), err)
}
}
}

Expand Down Expand Up @@ -374,10 +383,20 @@ func importNewRef(entry sortref.RefRevIdx, refStr string, opts *FlattenOpts) err
// At every iteration, new remotes may be found when digging deeper: they are rebased to the current schema before being imported.
//
// This returns true when no more remote references can be found.
// importableRefs returns every $ref that flatten has to bring into the root document: the ones the
// model maps, and the ones held by keywords it does not.
func importableRefs(sp *Spec) map[string]spec.Ref {
refs := make(map[string]spec.Ref, len(sp.references.schemas)+len(sp.references.unmappedRefs))
maps.Copy(refs, sp.references.schemas)
maps.Copy(refs, sp.references.unmappedRefs)

return refs
}

func importExternalReferences(opts *FlattenOpts) (bool, error) {
debugLog("importExternalReferences")

groupedRefs := sortref.ReverseIndex(opts.Spec.references.schemas, opts.BasePath)
groupedRefs := sortref.ReverseIndex(importableRefs(opts.Spec), opts.BasePath)
sortedRefStr := make([]string, 0, len(groupedRefs))
if opts.flattenContext == nil {
opts.flattenContext = newContext()
Expand Down
89 changes: 89 additions & 0 deletions flatten_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1463,3 +1463,92 @@ func testFlattenWithDefaults(t *testing.T, bp string) *Spec {

return an
}

func TestFlatten_UnmappedKeywordRef(t *testing.T) {
// A $ref under a keyword the Swagger 2.0 model does not map - propertyNames, if - lands in
// spec.Schema.ExtraProps as raw JSON. Flatten used to leave it alone while importing the
// subtree that held it, so the pointer named a definition of the root document, or nothing.
for _, testCase := range []struct {
name string
fixture string
expected string
assert func(t *testing.T, sp *spec.Swagger)
}{
{
name: "the target is imported once, and shared",
fixture: "root.json",
expected: "#/definitions/leaf",
assert: func(t *testing.T, sp *spec.Swagger) {
// the mapped $ref and the unmapped ones name the same imported definition
require.MapContainsT(t, sp.Definitions, "leaf")
mapped := sp.Definitions["deep"].Properties["mapped"]
assert.EqualT(t, "#/definitions/leaf", mapped.Ref.String())
},
},
{
name: "the target is imported although nothing mapped points at it",
fixture: "orphan-root.json",
expected: "#/definitions/leaf",
assert: func(t *testing.T, sp *spec.Swagger) {
require.MapContainsT(t, sp.Definitions, "leaf")
assert.TrueT(t, sp.Definitions["leaf"].Type.Contains("string"))
},
},
{
name: "a name conflict with the root does not capture the pointer",
fixture: "collide-root.json",
expected: "#/definitions/leafOAIGen",
assert: func(t *testing.T, sp *spec.Swagger) {
// the root keeps its own, unrelated "leaf"
assert.TrueT(t, sp.Definitions["leaf"].Type.Contains("integer"))
assert.TrueT(t, sp.Definitions["leafOAIGen"].Type.Contains("string"))
},
},
} {
t.Run(testCase.name, func(t *testing.T) {
bp := filepath.Join(".", "testdata", "unmapped", testCase.fixture)
sp := antest.LoadOrFail(t, bp)

require.NoError(t, Flatten(FlattenOpts{Spec: New(sp), BasePath: bp, Minimal: true}))

deep := sp.Definitions["deep"]
assert.EqualT(t, testCase.expected, unmappedRef(t, deep.ExtraProps["propertyNames"]))
testCase.assert(t, sp)

t.Run("the flattened document is self-contained", func(t *testing.T) {
for _, ref := range New(sp).AllRefs() {
assert.TrueT(t, strings.HasPrefix(ref.String(), "#/"), "expected a local $ref, got %q", ref)
}
assertUnmappedRefsResolve(t, sp)
})
})
}
}

// unmappedRef returns the $ref held by a raw JSON node.
func unmappedRef(t testing.TB, node any) string {
t.Helper()

asMap, ok := node.(map[string]any)
require.TrueT(t, ok)
ref, ok := asMap["$ref"].(string)
require.TrueT(t, ok)

return ref
}

// assertUnmappedRefsResolve checks that every $ref under an unmapped keyword names a definition
// that the flattened document actually holds.
func assertUnmappedRefsResolve(t *testing.T, sp *spec.Swagger) {
t.Helper()

an := New(sp)
require.NotEmpty(t, an.references.unmappedRefs)

for key := range an.references.unmappedRefs {
ref := an.references.unmappedRefs[key]
resolved, err := spec.ResolveRef(sp, &ref)
require.NoErrorf(t, err, "$ref %q at %s resolves to nothing", ref.String(), key)
require.NotNil(t, resolved)
}
}
20 changes: 10 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,23 @@ module github.com/go-openapi/analysis

require (
github.com/go-openapi/jsonpointer v1.0.0
github.com/go-openapi/spec v0.22.9
github.com/go-openapi/spec v0.22.10
github.com/go-openapi/strfmt v0.27.0
github.com/go-openapi/swag/jsonutils v0.29.0
github.com/go-openapi/swag/loading v0.29.0
github.com/go-openapi/swag/mangling v0.29.0
github.com/go-openapi/testify/v2 v2.6.1
github.com/go-openapi/swag/jsonutils v0.29.1
github.com/go-openapi/swag/loading v0.29.1
github.com/go-openapi/swag/mangling v0.29.1
github.com/go-openapi/testify/v2 v2.7.0
golang.org/x/text v0.41.0
)

require (
github.com/go-openapi/errors v0.22.8 // indirect
github.com/go-openapi/jsonreference v1.0.0 // indirect
github.com/go-openapi/swag/conv v0.29.0 // indirect
github.com/go-openapi/swag/pools v0.29.0 // indirect
github.com/go-openapi/swag/stringutils v0.28.0 // indirect
github.com/go-openapi/swag/typeutils v0.29.0 // indirect
github.com/go-openapi/swag/yamlutils v0.29.0 // indirect
github.com/go-openapi/swag/conv v0.29.1 // indirect
github.com/go-openapi/swag/pools v0.29.1 // indirect
github.com/go-openapi/swag/stringutils v0.29.1 // indirect
github.com/go-openapi/swag/typeutils v0.29.1 // indirect
github.com/go-openapi/swag/yamlutils v0.29.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/oklog/ulid/v2 v2.1.2 // indirect
Expand Down
44 changes: 22 additions & 22 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,32 @@ github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxg
github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y=
github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY=
github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0=
github.com/go-openapi/spec v0.22.9 h1:/vKIFDcGKp0ktZWGbym/tJEWbk6/XOEmAVU0kqKMH+w=
github.com/go-openapi/spec v0.22.9/go.mod h1:b/mNUYIOQOyIiUzUzXEE8xzyZqf93KvM9hQGP91yfl0=
github.com/go-openapi/spec v0.22.10 h1:5cp1dq++t4U/4WCg6f1wqReZowUrJ5kl8Eri4SMl51s=
github.com/go-openapi/spec v0.22.10/go.mod h1:aWRr+Ntv5tHoMQo0C1slTNLFo1FOYdZXFlUqViCN7yM=
github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM=
github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc=
github.com/go-openapi/swag/conv v0.29.0 h1:4+1TogWpOIzMPzVKrvx1BfqBYlApB7D7DW3EAWpwmp4=
github.com/go-openapi/swag/conv v0.29.0/go.mod h1:ch1l7V87F6zQXuLs5s0RFvrro6aFvrVcfVXn2PTZnu8=
github.com/go-openapi/swag/jsonutils v0.29.0 h1:Xgnf9g32ycQjQUnDxkhqLraH2FhitcSE3w7ayQB3TgA=
github.com/go-openapi/swag/jsonutils v0.29.0/go.mod h1:5WYmjf6hJcBve+ArzBaUsYy4M1GXsgjIQTmwJKfZHrA=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.0 h1:bpSF6LFkJJVtaRtJCzbZADVPVHQYKPwPdKthOQA2/5o=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.0/go.mod h1:julgTUKZ9/D0j6O7GKajmRs+812FWxQg/mMpGunWSjg=
github.com/go-openapi/swag/loading v0.29.0 h1:r1lg2DQbT1VgBwgiPYXBM059RNswFI6r36CC0QCcRGw=
github.com/go-openapi/swag/loading v0.29.0/go.mod h1:l/Z4MNbom0jSqzvWJqK2VUUWEceBknGEuVbLHLq4KN0=
github.com/go-openapi/swag/mangling v0.29.0 h1:RVKyucZ2rvA/M/sqxuNZGW8Mf0+1qypVX8n2GwV11lM=
github.com/go-openapi/swag/mangling v0.29.0/go.mod h1:SAop9pB7PUjQ/CGCNf/JmCKTRK+GDO+RqE9UHqC/N6s=
github.com/go-openapi/swag/pools v0.29.0 h1:uMQcoJeHJ8fWkdfEXJZMMpqk6hpfW8qTL5Q/IoRFFII=
github.com/go-openapi/swag/pools v0.29.0/go.mod h1:leDcaghjkRAhCuCRv9NfJU5f0mjoU3cT/XZObhMk3pc=
github.com/go-openapi/swag/stringutils v0.28.0 h1:ixsc9iYgDPubHL/8nSkbnryEHpD2VRlBMLKpQyPXcDU=
github.com/go-openapi/swag/stringutils v0.28.0/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM=
github.com/go-openapi/swag/typeutils v0.29.0 h1:HrWCYZeXVVNDo/7QQPRaYk33XeIDxksbxpalID3bWR8=
github.com/go-openapi/swag/typeutils v0.29.0/go.mod h1:hxpgDZJVBkBsi/d3MIUosafoFdE5exaQRmVp0zwu3YE=
github.com/go-openapi/swag/yamlutils v0.29.0 h1:JOKKuhMnBx4HYTM+kPEYw8S5YKKU9PnC4Mwb+c69BBA=
github.com/go-openapi/swag/yamlutils v0.29.0/go.mod h1:/+FVozjFWZzku6mRz5U/Qmq5Yk8PLFxBLLWA/jHaxYE=
github.com/go-openapi/swag/conv v0.29.1 h1:AC4Eh/5c/eUDOUCzzsRC9ghmFgOSBHeRMGIngY0ZUGA=
github.com/go-openapi/swag/conv v0.29.1/go.mod h1:S1X7/ZrBEZOC0Wc8AGxjbcGS92l3WEjA7aPtpl+RaqM=
github.com/go-openapi/swag/jsonutils v0.29.1 h1:AFCxs0eQZ24/QyfhVHM2t49rMz7Vv3XCsZQI6yrNy+c=
github.com/go-openapi/swag/jsonutils v0.29.1/go.mod h1:u3+sCfJpttDpcmS5kpm0yxL6GK0eWgODsx8Yw8fcqNM=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1 h1:BiiXE31Bx9SfpsMmOQj5KYpUhTZBpLVriVhJDuLuY2o=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1/go.mod h1:julgTUKZ9/D0j6O7GKajmRs+812FWxQg/mMpGunWSjg=
github.com/go-openapi/swag/loading v0.29.1 h1:FCv5fG8UhTdDJa2R7w+5O9Ekpcbw7tt0nFWvmDKGBjc=
github.com/go-openapi/swag/loading v0.29.1/go.mod h1:N0ESuem4p2oedKal8EJhciqnJ9Q9Wmt83L1CRB3Fouw=
github.com/go-openapi/swag/mangling v0.29.1 h1:lHALtvYCdxVnRl4GrHmFPwfBTZYIObqdGNSKyu/8D6I=
github.com/go-openapi/swag/mangling v0.29.1/go.mod h1:SAop9pB7PUjQ/CGCNf/JmCKTRK+GDO+RqE9UHqC/N6s=
github.com/go-openapi/swag/pools v0.29.1 h1:NRogYxdEW9SjRM4mkAOji9iefO4MRXq3p/ZJcoQbUKg=
github.com/go-openapi/swag/pools v0.29.1/go.mod h1:leDcaghjkRAhCuCRv9NfJU5f0mjoU3cT/XZObhMk3pc=
github.com/go-openapi/swag/stringutils v0.29.1 h1:1ykunK7iJQk1uOO7+oUH1ukbsK85fFCOiCFMOVSY+F0=
github.com/go-openapi/swag/stringutils v0.29.1/go.mod h1:7fSqZ+z8Qc0tOfAAK0jVa5qFGrnIlRi6n7NeGGrr1vc=
github.com/go-openapi/swag/typeutils v0.29.1 h1:Nzv9nhnlLCRBPQqfOX+7lB6Guju370or8StT+lIOf6M=
github.com/go-openapi/swag/typeutils v0.29.1/go.mod h1:hxpgDZJVBkBsi/d3MIUosafoFdE5exaQRmVp0zwu3YE=
github.com/go-openapi/swag/yamlutils v0.29.1 h1:69w3tsBajm7MR/fejLy7HD/3J68Ys1SeeZMEzZ3w2sk=
github.com/go-openapi/swag/yamlutils v0.29.1/go.mod h1:rgsp3vT/QdWzKwn43CigDwjOGIenPyTZMKnxEM8jZOA=
github.com/go-openapi/testify/enable/yaml/v2 v2.6.1 h1:Jm+/ze2rMtbD98yen92AhATGLGREDYXG56Xr4gMjEtE=
github.com/go-openapi/testify/enable/yaml/v2 v2.6.1/go.mod h1:YDPnwCRDu38/oJBVMBVXOUDiJ9cIeBHWvfImHaXqnv4=
github.com/go-openapi/testify/v2 v2.6.1 h1:6CNJhTjMzgaeaH8WhshcsZNPIvRemiOcFpU7seO/y7Q=
github.com/go-openapi/testify/v2 v2.6.1/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
github.com/go-openapi/testify/v2 v2.7.0 h1:bycOreEj6wfBvijg3YFogZ/sFjTCDmQnwSodSzHa3X8=
github.com/go-openapi/testify/v2 v2.7.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
Expand Down
12 changes: 12 additions & 0 deletions internal/flatten/replace/replace.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (

const (
definitionsPath = "#/definitions"
jsonRef = "$ref"
allocMediumMap = 64
)

Expand Down Expand Up @@ -217,6 +218,17 @@ func UpdateRef(sp any, key string, ref spec.Ref) error {
switch refable := value.(type) {
case *spec.Schema:
refable.Ref = ref
case map[string]any:
// a keyword the Swagger 2.0 model does not map: the node is raw JSON, and the map it
// holds is the one in the document - writing to it reaches the document
refable[jsonRef] = ref.String()
case *any:
raw, ok := (*refable).(map[string]any)
if !ok {
return ErrNoSchemaWithRef(key, value)
}

raw[jsonRef] = ref.String()
case *spec.SchemaOrArray:
if refable.Schema != nil {
refable.Schema.Ref = ref
Expand Down
Loading
Loading