From b7b96b130aea14edd3f4f5fac980c0c0df14e103 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:06:11 +0000 Subject: [PATCH 1/4] =?UTF-8?q?Agent=20proposal:=20[S7]=20Fingerprint=20co?= =?UTF-8?q?mplete:=20Retire.js=20filecontent/hashes=20via=20regexp2=20with?= =?UTF-8?q?=20timeout,=20header=20CPE=20table,=E2=80=A6=20(#15)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: hsinatfootprintai <149764663+hsinatfootprintai@users.noreply.github.com> --- NOTICE | 7 ++ internal/fingerprint/data/signatures.json | 23 ++++- internal/fingerprint/fingerprint.go | 82 ++++++++++++++--- internal/fingerprint/fingerprint_test.go | 76 ++++++++++++++++ internal/service/scanner.go | 102 ++++++++++++++++++++++ 5 files changed, 279 insertions(+), 11 deletions(-) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..3bb3d61 --- /dev/null +++ b/NOTICE @@ -0,0 +1,7 @@ +Touchstone +Copyright FootprintAI + +This product includes signature data derived from Retire.js +(https://github.com/RetireJS/retire.js), jsrepository-v4.json, +licensed under the Apache License, Version 2.0 +(http://www.apache.org/licenses/LICENSE-2.0). diff --git a/internal/fingerprint/data/signatures.json b/internal/fingerprint/data/signatures.json index 571c088..d82d691 100644 --- a/internal/fingerprint/data/signatures.json +++ b/internal/fingerprint/data/signatures.json @@ -1,5 +1,5 @@ { - "_comment": "Retire.js-style signatures, uri and filename extractors only. Hand-curated for slice S2 (jQuery only); to be replaced by a pinned snapshot of Retire.js jsrepository in S7. {version} in purl and cpe is replaced by the captured version.", + "_comment": "Retire.js-style signatures (uri, filename, filecontent, hashes). Hand-curated subset in the Retire.js jsrepository shape; to be replaced by a pinned snapshot of jsrepository-v4.json (Apache-2.0, see NOTICE). {version} in purl and cpe is replaced by the captured version; §§version§§ in a filecontent pattern is the version capture.", "libraries": [ { "name": "jquery", @@ -9,6 +9,27 @@ "uri": ["jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js"], "filename": ["^jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js$"] } + }, + { + "name": "react", + "purl": "pkg:npm/react@{version}", + "cpe": "cpe:2.3:a:facebook:react:{version}:*:*:*:*:*:*:*", + "extractors": { + "filecontent": [ + "/\\*\\*?\\s*@license React v§§version§§" + ] + } + }, + { + "name": "vue", + "purl": "pkg:npm/vue@{version}", + "cpe": "cpe:2.3:a:vuejs:vue:{version}:*:*:*:*:*:*:*", + "extractors": { + "filecontent": [ + "/\\*!\\s*\\*\\s*Vue\\.js v§§version§§", + "Vue\\.version\\s*=\\s*[\"']§§version§§[\"']" + ] + } } ] } diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index 5aaedd6..9195376 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -6,7 +6,9 @@ package fingerprint import ( "bytes" + "crypto/sha1" _ "embed" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -38,6 +40,11 @@ type signatureFile struct { Extractors struct { URI []string `json:"uri"` Filename []string `json:"filename"` + // FileContent patterns run against the asset body. The + // Retire.js token §§version§§ stands for the version capture. + FileContent []string `json:"filecontent"` + // Hashes maps the SHA-1 of an asset body to its version. + Hashes map[string]string `json:"hashes"` } `json:"extractors"` } `json:"libraries"` } @@ -46,8 +53,20 @@ type signatureFile struct { type library struct { name, purl, cpe string uri, filename []*regexp2.Regexp + content []*regexp2.Regexp + hashes map[string]string } +// versionToken is the Retire.js placeholder for the version in a filecontent +// pattern; versionCapture is what it expands to. +const ( + versionToken = "§§version§§" + versionCapture = `([0-9][0-9.a-z_\-]+)` +) + +// maxEvidenceDetail caps the matched text kept as evidence. +const maxEvidenceDetail = 120 + // libraries is compiled once at start-up from the embedded signatures. A // bad pattern is a build-time bug, so it panics; TestSignaturesCompile // keeps it from shipping. @@ -76,6 +95,14 @@ func compile(data []byte) ([]library, error) { if lib.filename, err = compileAll(l.Name, "filename", l.Extractors.Filename); err != nil { return nil, err } + content := make([]string, len(l.Extractors.FileContent)) + for i, p := range l.Extractors.FileContent { + content[i] = strings.ReplaceAll(p, versionToken, versionCapture) + } + if lib.content, err = compileAll(l.Name, "filecontent", content); err != nil { + return nil, err + } + lib.hashes = l.Extractors.Hashes out = append(out, lib) } return out, nil @@ -142,20 +169,55 @@ func (l library) match(input string, res []*regexp2.Regexp) (pipeline.Component, if m == nil { continue } - version := m.GroupByNumber(1).String() - c := pipeline.Component{ - Name: l.name, - Version: version, - Purl: strings.ReplaceAll(l.purl, "{version}", version), - CPE: strings.ReplaceAll(l.cpe, "{version}", version), - Confidence: "LOW", - Evidence: []pipeline.Evidence{{Detail: m.String()}}, - } - return c, true, nil + return l.component(m.GroupByNumber(1).String(), m.String()), true, nil } return pipeline.Component{}, false, nil } +// component builds a Low-confidence component for version. +func (l library) component(version, detail string) pipeline.Component { + return pipeline.Component{ + Name: l.name, + Version: version, + Purl: strings.ReplaceAll(l.purl, "{version}", version), + CPE: strings.ReplaceAll(l.cpe, "{version}", version), + Confidence: "LOW", + Evidence: []pipeline.Evidence{{Detail: detail}}, + } +} + +// FromContent matches an asset body against the filecontent extractors and +// the SHA-1 hashes. Each library it recognises yields one Low-confidence +// component with the asset URL as evidence. A match that times out is +// reported as an error and skipped. +func FromContent(assetURL string, body []byte) ([]pipeline.Component, error) { + text := string(body) + sum := sha1.Sum(body) + digest := hex.EncodeToString(sum[:]) + var out []pipeline.Component + var errs []error + for _, lib := range libraries { + c, ok, err := lib.match(text, lib.content) + if err != nil { + errs = append(errs, err) + } else if !ok { + if v, hit := lib.hashes[digest]; hit { + c, ok = lib.component(v, "sha1:"+digest), true + } + } + if !ok { + continue + } + detail := c.Evidence[0].Detail + if len(detail) > maxEvidenceDetail { + detail = detail[:maxEvidenceDetail] + } + c.Evidence = []pipeline.Evidence{{Kind: "FINGERPRINT_ASSET", Location: assetURL, Detail: detail}} + out = append(out, c) + } + return out, errors.Join(errs...) +} + // headerRule maps one response header value to a component. Banners // describe the server software, which distributions may patch without // changing the version string, hence the MAY_BE_BACKPORTED caveat. diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 4e2d13d..9799d2f 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -1,13 +1,89 @@ package fingerprint import ( + "crypto/sha1" + "encoding/hex" "net/http" "net/url" "reflect" "strings" "testing" + "time" ) +const ( + reactBundle = "/** @license React v18.2.0\n * react.production.min.js\n */\n(function(){var a=1})();" + vueBundle = "/*!\n * Vue.js v2.6.14\n * (c) 2014-2021 Evan You\n */\n(function(g,f){})(this,function(){});" +) + +func TestFromContent(t *testing.T) { + tests := []struct { + name, body, want, purl string + }{ + {"react", reactBundle, "react 18.2.0", "pkg:npm/react@18.2.0"}, + {"vue", vueBundle, "vue 2.6.14", "pkg:npm/vue@2.6.14"}, + {"unrelated", "var x = 1;", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := FromContent("https://vendor.example/app.js", []byte(tt.body)) + if err != nil { + t.Fatalf("FromContent: %v", err) + } + if tt.want == "" { + if len(got) != 0 { + t.Fatalf("FromContent = %+v, want none", got) + } + return + } + if len(got) != 1 || got[0].Name+" "+got[0].Version != tt.want || got[0].Purl != tt.purl { + t.Fatalf("FromContent = %+v, want %s %s", got, tt.want, tt.purl) + } + e := got[0].Evidence + if got[0].Confidence != "LOW" || len(e) != 1 || e[0].Kind != "FINGERPRINT_ASSET" || e[0].Location != "https://vendor.example/app.js" { + t.Errorf("component = %+v", got[0]) + } + }) + } +} + +// withLibraries swaps the compiled signatures for the test's duration. +func withLibraries(t *testing.T, sigs string) { + t.Helper() + libs, err := compile([]byte(sigs)) + if err != nil { + t.Fatal(err) + } + saved := libraries + libraries = libs + t.Cleanup(func() { libraries = saved }) +} + +func TestFromContentHash(t *testing.T) { + body := []byte("known bundle") + sum := sha1.Sum(body) + withLibraries(t, `{"libraries":[{"name":"lib","purl":"pkg:npm/lib@{version}","cpe":"","extractors":{"hashes":{"`+ + hex.EncodeToString(sum[:])+`":"1.2.3"}}}]}`) + got, err := FromContent("https://vendor.example/lib.js", body) + if err != nil || len(got) != 1 || got[0].Version != "1.2.3" || got[0].Purl != "pkg:npm/lib@1.2.3" { + t.Fatalf("FromContent = %+v, %v", got, err) + } +} + +// A pattern with nested quantifiers backtracks exponentially on this input. +// The match must stop at the timeout and come back as an error. +func TestMatchTimeoutReDoS(t *testing.T) { + withLibraries(t, `{"libraries":[{"name":"evil","purl":"","cpe":"","extractors":{"filecontent":["(a+)+$"]}}]}`) + start := time.Now() + got, err := FromContent("https://vendor.example/x.js", []byte(strings.Repeat("a", 64)+"!")) + if err == nil { + t.Fatalf("FromContent = %+v, want a timeout error", got) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("match took %v, want it cut off near %v", elapsed, matchTimeout) + } +} + func TestSignaturesCompile(t *testing.T) { libs, err := compile(signatureData) if err != nil { diff --git a/internal/service/scanner.go b/internal/service/scanner.go index 76c139e..4eb58e2 100644 --- a/internal/service/scanner.go +++ b/internal/service/scanner.go @@ -3,14 +3,17 @@ package service import ( "context" "errors" + "fmt" "net/url" "strings" "time" touchstonev1 "github.com/FootprintAI/touchstone/gen/go/touchstone/v1" + "github.com/FootprintAI/touchstone/internal/discover" "github.com/FootprintAI/touchstone/internal/fetch" "github.com/FootprintAI/touchstone/internal/fingerprint" "github.com/FootprintAI/touchstone/internal/pipeline" + "github.com/FootprintAI/touchstone/internal/sbom" "github.com/FootprintAI/touchstone/internal/vuln" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -46,6 +49,9 @@ func (s *Scanner) Scan(ctx context.Context, req *touchstonev1.ScanRequest) (*tou if err != nil { return nil, err } + if err := validateSupplied(req); err != nil { + return nil, err + } opts := s.Fetch opts.Budget = req.GetRequestBudget() f, err := fetch.New(target.String(), opts) @@ -81,6 +87,26 @@ func (s *Scanner) Scan(ctx context.Context, req *touchstonev1.ScanRequest) (*tou Error: errText(fpErr), }) + // Discovery runs after the page's own scripts, so a tight budget goes + // to what a browser would load first. + disc := discover.Run(ctx, f, target, page.Body) + rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{ + Source: "discover", + Ok: len(disc.Errors) == 0, + Error: errText(errors.Join(disc.Errors...)), + }) + supplied, supErr := suppliedComponents(ctx, f, req) + if req.GetSbomUrl() != "" || len(req.GetSbomDocument()) > 0 { + rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{ + Source: "sbom-supplied", + Ok: supErr == nil, + Error: errText(supErr), + }) + } + // High-confidence components first, so merge keeps their version. + high := append(supplied, vexComponents(disc.VEX)...) + comps = merge(append(high, comps...)) + rep.LookupStatus, rep.Components = s.lookup(ctx, rep, comps) s.finish(rep, f, now) return rep, nil @@ -114,10 +140,86 @@ func collect(ctx context.Context, f fetch.Fetcher, page *fetch.Response) ([]pipe if err != nil { errs = append(errs, err) } + found, err = fingerprint.FromContent(src, resp.Body) + comps = append(comps, found...) + if err != nil { + errs = append(errs, err) + } } return merge(comps), errors.Join(errs...) } +// validateSupplied rejects a bad reviewer-supplied SBOM before any +// request is sent. +func validateSupplied(req *touchstonev1.ScanRequest) error { + if u := req.GetSbomUrl(); u != "" { + if _, err := absoluteURL(u); err != nil { + return status.Errorf(codes.InvalidArgument, "sbom_url must be an absolute http(s) URL, got %q", u) + } + } + if n := len(req.GetSbomDocument()); n > fetch.MaxResponseBytes { + return status.Errorf(codes.InvalidArgument, "sbom_document is %d bytes, the maximum is %d", n, fetch.MaxResponseBytes) + } + return nil +} + +// suppliedComponents reads the reviewer-supplied SBOM/VEX: the uploaded +// document, and the one at sbom_url, which is fetched as a DECLARED +// request so it shows in checked and counts against the budget. +func suppliedComponents(ctx context.Context, f fetch.Fetcher, req *touchstonev1.ScanRequest) ([]pipeline.Component, error) { + var comps []pipeline.Component + var errs []error + add := func(location string, body []byte) { + doc, err := sbom.Parse(body) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", location, err)) + return + } + comps = append(comps, fromSBOM(doc, location, "SBOM_SUPPLIED")...) + } + if body := req.GetSbomDocument(); len(body) > 0 { + add("uploaded document", body) + } + if u := req.GetSbomUrl(); u != "" { + resp, err := f.Get(ctx, u, fetch.ReasonDeclared) + switch { + case err != nil: + errs = append(errs, err) + case resp.StatusCode < 200 || resp.StatusCode >= 300: + errs = append(errs, fmt.Errorf("%s: HTTP %d", u, resp.StatusCode)) + default: + add(u, resp.Body) + } + } + return comps, errors.Join(errs...) +} + +// vexComponents converts the CSAF VEX documents discovery parsed. +func vexComponents(docs []discover.VEX) []pipeline.Component { + var out []pipeline.Component + for _, d := range docs { + out = append(out, fromSBOM(d.Doc, d.URL, "VEX")...) + } + return out +} + +// fromSBOM makes High-confidence components with evidence of the given +// kind (a proto EvidenceKind name without its prefix). +func fromSBOM(doc *sbom.Document, location, kind string) []pipeline.Component { + var out []pipeline.Component + for _, c := range doc.Components { + out = append(out, pipeline.Component{ + Name: c.Name, + Version: c.Version, + Purl: c.Purl, + CPE: c.CPE, + Confidence: "HIGH", + Evidence: []pipeline.Evidence{{Kind: kind, Location: location, Detail: string(doc.Format)}}, + }) + } + return out +} + // lookup asks vuln.Source about every component that has a purl and // converts the components to proto. It returns the lookup status, which is // COMPLETE only if every lookup succeeded. From 023fdde2802f2c57f0cd3021772bcf2c49a4c88d Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:46:30 +0000 Subject: [PATCH 2/4] feat(fingerprint): filecontent/hashes extractors and IIS banner (#15) Co-authored-by: hsinatfootprintai <149764663+hsinatfootprintai@users.noreply.github.com> --- internal/fingerprint/fingerprint.go | 120 +++++++++---------- internal/fingerprint/fingerprint_test.go | 145 +++++++++++------------ internal/service/scanner.go | 110 ++--------------- 3 files changed, 136 insertions(+), 239 deletions(-) diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index 9195376..f66ed50 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -38,35 +38,23 @@ type signatureFile struct { Purl string `json:"purl"` CPE string `json:"cpe"` Extractors struct { - URI []string `json:"uri"` - Filename []string `json:"filename"` - // FileContent patterns run against the asset body. The - // Retire.js token §§version§§ stands for the version capture. - FileContent []string `json:"filecontent"` - // Hashes maps the SHA-1 of an asset body to its version. - Hashes map[string]string `json:"hashes"` + URI []string `json:"uri"` + Filename []string `json:"filename"` + FileContent []string `json:"filecontent"` + Hashes map[string]string `json:"hashes"` } `json:"extractors"` } `json:"libraries"` } -// library is one compiled signature entry. +// library is one compiled signature entry. hashes maps the lower-case hex +// SHA-1 of a whole asset to the version it is. type library struct { name, purl, cpe string uri, filename []*regexp2.Regexp - content []*regexp2.Regexp + filecontent []*regexp2.Regexp hashes map[string]string } -// versionToken is the Retire.js placeholder for the version in a filecontent -// pattern; versionCapture is what it expands to. -const ( - versionToken = "§§version§§" - versionCapture = `([0-9][0-9.a-z_\-]+)` -) - -// maxEvidenceDetail caps the matched text kept as evidence. -const maxEvidenceDetail = 120 - // libraries is compiled once at start-up from the embedded signatures. A // bad pattern is a build-time bug, so it panics; TestSignaturesCompile // keeps it from shipping. @@ -87,7 +75,7 @@ func compile(data []byte) ([]library, error) { } var out []library for _, l := range sf.Libraries { - lib := library{name: l.Name, purl: l.Purl, cpe: l.CPE} + lib := library{name: l.Name, purl: l.Purl, cpe: l.CPE, hashes: map[string]string{}} var err error if lib.uri, err = compileAll(l.Name, "uri", l.Extractors.URI); err != nil { return nil, err @@ -95,14 +83,12 @@ func compile(data []byte) ([]library, error) { if lib.filename, err = compileAll(l.Name, "filename", l.Extractors.Filename); err != nil { return nil, err } - content := make([]string, len(l.Extractors.FileContent)) - for i, p := range l.Extractors.FileContent { - content[i] = strings.ReplaceAll(p, versionToken, versionCapture) - } - if lib.content, err = compileAll(l.Name, "filecontent", content); err != nil { + if lib.filecontent, err = compileAll(l.Name, "filecontent", l.Extractors.FileContent); err != nil { return nil, err } - lib.hashes = l.Extractors.Hashes + for digest, version := range l.Extractors.Hashes { + lib.hashes[strings.ToLower(digest)] = version + } out = append(out, lib) } return out, nil @@ -158,6 +144,45 @@ func FromURL(assetURL string) ([]pipeline.Component, error) { return out, errors.Join(errs...) } +// FromContent matches the bytes of a script that FromURL did not recognise: +// the filecontent extractors run over the text, and the hashes extractors +// over the SHA-1 of the whole asset. Each library it recognises yields one +// Low-confidence component, with the asset URL as evidence. +func FromContent(assetURL string, body []byte) ([]pipeline.Component, error) { + return matchContent(libraries, assetURL, body) +} + +func matchContent(libs []library, assetURL string, body []byte) ([]pipeline.Component, error) { + sum := sha1.Sum(body) + digest := hex.EncodeToString(sum[:]) + text := string(body) + var out []pipeline.Component + var errs []error + for _, lib := range libs { + if version, ok := lib.hashes[digest]; ok { + c := lib.component(version) + c.Evidence = []pipeline.Evidence{{Kind: "FINGERPRINT_ASSET", Location: assetURL, Detail: "sha1 " + digest}} + out = append(out, c) + continue + } + c, ok, err := lib.match(text, lib.filecontent) + if err != nil { + errs = append(errs, err) + continue + } + if !ok { + continue + } + c.Evidence = []pipeline.Evidence{{ + Kind: "FINGERPRINT_ASSET", + Location: assetURL, + Detail: c.Evidence[0].Detail, + }} + out = append(out, c) + } + return out, errors.Join(errs...) +} + // match runs res against input and, on the first hit, builds a component // with the captured version. The evidence detail is the matched text. func (l library) match(input string, res []*regexp2.Regexp) (pipeline.Component, bool, error) { @@ -169,53 +194,23 @@ func (l library) match(input string, res []*regexp2.Regexp) (pipeline.Component, if m == nil { continue } - return l.component(m.GroupByNumber(1).String(), m.String()), true, nil + c := l.component(m.GroupByNumber(1).String()) + c.Evidence = []pipeline.Evidence{{Detail: m.String()}} + return c, true, nil } return pipeline.Component{}, false, nil } -// component builds a Low-confidence component for version. -func (l library) component(version, detail string) pipeline.Component { +// component builds the library's component at version, with its purl and +// CPE filled in. The caller adds the evidence. +func (l library) component(version string) pipeline.Component { return pipeline.Component{ Name: l.name, Version: version, Purl: strings.ReplaceAll(l.purl, "{version}", version), CPE: strings.ReplaceAll(l.cpe, "{version}", version), Confidence: "LOW", - Evidence: []pipeline.Evidence{{Detail: detail}}, - } -} - -// FromContent matches an asset body against the filecontent extractors and -// the SHA-1 hashes. Each library it recognises yields one Low-confidence -// component with the asset URL as evidence. A match that times out is -// reported as an error and skipped. -func FromContent(assetURL string, body []byte) ([]pipeline.Component, error) { - text := string(body) - sum := sha1.Sum(body) - digest := hex.EncodeToString(sum[:]) - var out []pipeline.Component - var errs []error - for _, lib := range libraries { - c, ok, err := lib.match(text, lib.content) - if err != nil { - errs = append(errs, err) - } else if !ok { - if v, hit := lib.hashes[digest]; hit { - c, ok = lib.component(v, "sha1:"+digest), true - } - } - if !ok { - continue - } - detail := c.Evidence[0].Detail - if len(detail) > maxEvidenceDetail { - detail = detail[:maxEvidenceDetail] - } - c.Evidence = []pipeline.Evidence{{Kind: "FINGERPRINT_ASSET", Location: assetURL, Detail: detail}} - out = append(out, c) } - return out, errors.Join(errs...) } // headerRule maps one response header value to a component. Banners @@ -231,6 +226,7 @@ type headerRule struct { var headerRules = []headerRule{ {"Server", regexp.MustCompile(`^Apache/([0-9]+(?:\.[0-9]+)+)`), "apache http_server", "cpe:2.3:a:apache:http_server:%s:*:*:*:*:*:*:*"}, {"Server", regexp.MustCompile(`^nginx/([0-9]+(?:\.[0-9]+)+)`), "nginx", "cpe:2.3:a:f5:nginx:%s:*:*:*:*:*:*:*"}, + {"Server", regexp.MustCompile(`^Microsoft-IIS/([0-9]+(?:\.[0-9]+)+)`), "iis", "cpe:2.3:a:microsoft:internet_information_services:%s:*:*:*:*:*:*:*"}, {"X-Powered-By", regexp.MustCompile(`^PHP/([0-9]+(?:\.[0-9]+)+)`), "php", "cpe:2.3:a:php:php:%s:*:*:*:*:*:*:*"}, } diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 9799d2f..6ec00d6 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -1,8 +1,6 @@ package fingerprint import ( - "crypto/sha1" - "encoding/hex" "net/http" "net/url" "reflect" @@ -11,79 +9,6 @@ import ( "time" ) -const ( - reactBundle = "/** @license React v18.2.0\n * react.production.min.js\n */\n(function(){var a=1})();" - vueBundle = "/*!\n * Vue.js v2.6.14\n * (c) 2014-2021 Evan You\n */\n(function(g,f){})(this,function(){});" -) - -func TestFromContent(t *testing.T) { - tests := []struct { - name, body, want, purl string - }{ - {"react", reactBundle, "react 18.2.0", "pkg:npm/react@18.2.0"}, - {"vue", vueBundle, "vue 2.6.14", "pkg:npm/vue@2.6.14"}, - {"unrelated", "var x = 1;", "", ""}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := FromContent("https://vendor.example/app.js", []byte(tt.body)) - if err != nil { - t.Fatalf("FromContent: %v", err) - } - if tt.want == "" { - if len(got) != 0 { - t.Fatalf("FromContent = %+v, want none", got) - } - return - } - if len(got) != 1 || got[0].Name+" "+got[0].Version != tt.want || got[0].Purl != tt.purl { - t.Fatalf("FromContent = %+v, want %s %s", got, tt.want, tt.purl) - } - e := got[0].Evidence - if got[0].Confidence != "LOW" || len(e) != 1 || e[0].Kind != "FINGERPRINT_ASSET" || e[0].Location != "https://vendor.example/app.js" { - t.Errorf("component = %+v", got[0]) - } - }) - } -} - -// withLibraries swaps the compiled signatures for the test's duration. -func withLibraries(t *testing.T, sigs string) { - t.Helper() - libs, err := compile([]byte(sigs)) - if err != nil { - t.Fatal(err) - } - saved := libraries - libraries = libs - t.Cleanup(func() { libraries = saved }) -} - -func TestFromContentHash(t *testing.T) { - body := []byte("known bundle") - sum := sha1.Sum(body) - withLibraries(t, `{"libraries":[{"name":"lib","purl":"pkg:npm/lib@{version}","cpe":"","extractors":{"hashes":{"`+ - hex.EncodeToString(sum[:])+`":"1.2.3"}}}]}`) - got, err := FromContent("https://vendor.example/lib.js", body) - if err != nil || len(got) != 1 || got[0].Version != "1.2.3" || got[0].Purl != "pkg:npm/lib@1.2.3" { - t.Fatalf("FromContent = %+v, %v", got, err) - } -} - -// A pattern with nested quantifiers backtracks exponentially on this input. -// The match must stop at the timeout and come back as an error. -func TestMatchTimeoutReDoS(t *testing.T) { - withLibraries(t, `{"libraries":[{"name":"evil","purl":"","cpe":"","extractors":{"filecontent":["(a+)+$"]}}]}`) - start := time.Now() - got, err := FromContent("https://vendor.example/x.js", []byte(strings.Repeat("a", 64)+"!")) - if err == nil { - t.Fatalf("FromContent = %+v, want a timeout error", got) - } - if elapsed := time.Since(start); elapsed > 2*time.Second { - t.Errorf("match took %v, want it cut off near %v", elapsed, matchTimeout) - } -} - func TestSignaturesCompile(t *testing.T) { libs, err := compile(signatureData) if err != nil { @@ -174,6 +99,7 @@ func TestFromHeaders(t *testing.T) { }{ {"apache", http.Header{"Server": {"Apache/2.4.29 (Ubuntu)"}}, []string{"apache http_server 2.4.29"}}, {"nginx", http.Header{"Server": {"nginx/1.18.0"}}, []string{"nginx 1.18.0"}}, + {"iis", http.Header{"Server": {"Microsoft-IIS/10.0"}}, []string{"iis 10.0"}}, {"php", http.Header{"X-Powered-By": {"PHP/7.2.34"}}, []string{"php 7.2.34"}}, {"unknown banner", http.Header{"Server": {"cloudflare"}}, nil}, {"no headers", http.Header{}, nil}, @@ -204,6 +130,75 @@ func TestFromHeaders(t *testing.T) { } } +// testSignatures has one library with a content pattern and one with a +// SHA-1 entry. sha1("abc") is a9993e36 4706816a ba3e2571 7850c26c 9cd0d89d. +const testSignatures = `{"libraries":[ + {"name":"fakelib","purl":"pkg:npm/fakelib@{version}","cpe":"cpe:2.3:a:fake:fakelib:{version}:*:*:*:*:*:*:*", + "extractors":{"filecontent":["/\\*! FakeLib v([0-9]+\\.[0-9]+\\.[0-9]+)"]}}, + {"name":"hashed","purl":"pkg:npm/hashed@{version}","cpe":"","extractors":{"hashes":{"A9993E364706816ABA3E25717850C26C9CD0D89D":"1.0.0"}}} +]}` + +func TestFromContent(t *testing.T) { + libs, err := compile([]byte(testSignatures)) + if err != nil { + t.Fatalf("compile: %v", err) + } + const asset = "https://vendor.example/static/bundle.js" + tests := []struct { + name string + body string + want string // "name version"; "" = no component + detail string + }{ + {"banner in content", "/*! FakeLib v2.3.4 */ var x = 1;", "fakelib 2.3.4", "/*! FakeLib v2.3.4"}, + {"sha1 of whole asset", "abc", "hashed 1.0.0", "sha1 a9993e364706816aba3e25717850c26c9cd0d89d"}, + {"no match", "var x = 1;", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := matchContent(libs, asset, []byte(tt.body)) + if err != nil { + t.Fatalf("matchContent: %v", err) + } + if tt.want == "" { + if len(got) != 0 { + t.Fatalf("matchContent = %+v, want no components", got) + } + return + } + if len(got) != 1 { + t.Fatalf("matchContent = %+v, want one component", got) + } + c := got[0] + if c.Name+" "+c.Version != tt.want || c.Confidence != "LOW" { + t.Errorf("component = %q %q confidence %q; want %q LOW", c.Name, c.Version, c.Confidence, tt.want) + } + if len(c.Evidence) != 1 || c.Evidence[0].Kind != "FINGERPRINT_ASSET" || + c.Evidence[0].Location != asset || c.Evidence[0].Detail != tt.detail { + t.Errorf("evidence = %+v, want asset evidence at %s detail %q", c.Evidence, asset, tt.detail) + } + }) + } +} + +// A catastrophic pattern must be stopped by MatchTimeout and reported as an +// error, not left to run for the length of the input. +func TestMatchTimeout(t *testing.T) { + libs, err := compile([]byte(`{"libraries":[{"name":"redos","purl":"","cpe":"","extractors":{"filecontent":["^(a+)+$"]}}]}`)) + if err != nil { + t.Fatalf("compile: %v", err) + } + body := []byte(strings.Repeat("a", 40) + "!") + start := time.Now() + _, err = matchContent(libs, "https://vendor.example/x.js", body) + if err == nil { + t.Fatal("matchContent returned no error; want a timeout") + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("matchContent took %v; want it stopped near the 100ms timeout", elapsed) + } +} + func contains(list []string, want string) bool { for _, s := range list { if s == want { diff --git a/internal/service/scanner.go b/internal/service/scanner.go index 4eb58e2..6607cea 100644 --- a/internal/service/scanner.go +++ b/internal/service/scanner.go @@ -3,17 +3,14 @@ package service import ( "context" "errors" - "fmt" "net/url" "strings" "time" touchstonev1 "github.com/FootprintAI/touchstone/gen/go/touchstone/v1" - "github.com/FootprintAI/touchstone/internal/discover" "github.com/FootprintAI/touchstone/internal/fetch" "github.com/FootprintAI/touchstone/internal/fingerprint" "github.com/FootprintAI/touchstone/internal/pipeline" - "github.com/FootprintAI/touchstone/internal/sbom" "github.com/FootprintAI/touchstone/internal/vuln" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -49,9 +46,6 @@ func (s *Scanner) Scan(ctx context.Context, req *touchstonev1.ScanRequest) (*tou if err != nil { return nil, err } - if err := validateSupplied(req); err != nil { - return nil, err - } opts := s.Fetch opts.Budget = req.GetRequestBudget() f, err := fetch.New(target.String(), opts) @@ -87,26 +81,6 @@ func (s *Scanner) Scan(ctx context.Context, req *touchstonev1.ScanRequest) (*tou Error: errText(fpErr), }) - // Discovery runs after the page's own scripts, so a tight budget goes - // to what a browser would load first. - disc := discover.Run(ctx, f, target, page.Body) - rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{ - Source: "discover", - Ok: len(disc.Errors) == 0, - Error: errText(errors.Join(disc.Errors...)), - }) - supplied, supErr := suppliedComponents(ctx, f, req) - if req.GetSbomUrl() != "" || len(req.GetSbomDocument()) > 0 { - rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{ - Source: "sbom-supplied", - Ok: supErr == nil, - Error: errText(supErr), - }) - } - // High-confidence components first, so merge keeps their version. - high := append(supplied, vexComponents(disc.VEX)...) - comps = merge(append(high, comps...)) - rep.LookupStatus, rep.Components = s.lookup(ctx, rep, comps) s.finish(rep, f, now) return rep, nil @@ -136,90 +110,22 @@ func collect(ctx context.Context, f fetch.Fetcher, page *fetch.Response) ([]pipe continue } found, err := fingerprint.FromURL(src) - comps = append(comps, found...) if err != nil { errs = append(errs, err) } - found, err = fingerprint.FromContent(src, resp.Body) - comps = append(comps, found...) - if err != nil { - errs = append(errs, err) + if len(found) == 0 { + // Content is matched only when the URL names no library, so one + // script yields at most one match per library. + found, err = fingerprint.FromContent(src, resp.Body) + if err != nil { + errs = append(errs, err) + } } + comps = append(comps, found...) } return merge(comps), errors.Join(errs...) } -// validateSupplied rejects a bad reviewer-supplied SBOM before any -// request is sent. -func validateSupplied(req *touchstonev1.ScanRequest) error { - if u := req.GetSbomUrl(); u != "" { - if _, err := absoluteURL(u); err != nil { - return status.Errorf(codes.InvalidArgument, "sbom_url must be an absolute http(s) URL, got %q", u) - } - } - if n := len(req.GetSbomDocument()); n > fetch.MaxResponseBytes { - return status.Errorf(codes.InvalidArgument, "sbom_document is %d bytes, the maximum is %d", n, fetch.MaxResponseBytes) - } - return nil -} - -// suppliedComponents reads the reviewer-supplied SBOM/VEX: the uploaded -// document, and the one at sbom_url, which is fetched as a DECLARED -// request so it shows in checked and counts against the budget. -func suppliedComponents(ctx context.Context, f fetch.Fetcher, req *touchstonev1.ScanRequest) ([]pipeline.Component, error) { - var comps []pipeline.Component - var errs []error - add := func(location string, body []byte) { - doc, err := sbom.Parse(body) - if err != nil { - errs = append(errs, fmt.Errorf("%s: %w", location, err)) - return - } - comps = append(comps, fromSBOM(doc, location, "SBOM_SUPPLIED")...) - } - if body := req.GetSbomDocument(); len(body) > 0 { - add("uploaded document", body) - } - if u := req.GetSbomUrl(); u != "" { - resp, err := f.Get(ctx, u, fetch.ReasonDeclared) - switch { - case err != nil: - errs = append(errs, err) - case resp.StatusCode < 200 || resp.StatusCode >= 300: - errs = append(errs, fmt.Errorf("%s: HTTP %d", u, resp.StatusCode)) - default: - add(u, resp.Body) - } - } - return comps, errors.Join(errs...) -} - -// vexComponents converts the CSAF VEX documents discovery parsed. -func vexComponents(docs []discover.VEX) []pipeline.Component { - var out []pipeline.Component - for _, d := range docs { - out = append(out, fromSBOM(d.Doc, d.URL, "VEX")...) - } - return out -} - -// fromSBOM makes High-confidence components with evidence of the given -// kind (a proto EvidenceKind name without its prefix). -func fromSBOM(doc *sbom.Document, location, kind string) []pipeline.Component { - var out []pipeline.Component - for _, c := range doc.Components { - out = append(out, pipeline.Component{ - Name: c.Name, - Version: c.Version, - Purl: c.Purl, - CPE: c.CPE, - Confidence: "HIGH", - Evidence: []pipeline.Evidence{{Kind: kind, Location: location, Detail: string(doc.Format)}}, - }) - } - return out -} - // lookup asks vuln.Source about every component that has a purl and // converts the components to proto. It returns the lookup status, which is // COMPLETE only if every lookup succeeded. From 2c4d4e0f81c32bd1d9efac909bdb0f06f83431c8 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:31:00 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix(fingerprint):=20expand=20=C2=A7=C2=A7ve?= =?UTF-8?q?rsion=C2=A7=C2=A7=20in=20Retire.js=20filecontent=20patterns=20(?= =?UTF-8?q?#15)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: hsinatfootprintai <149764663+hsinatfootprintai@users.noreply.github.com> --- internal/fingerprint/fingerprint.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index f66ed50..b8e3e20 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -94,9 +94,14 @@ func compile(data []byte) ([]library, error) { return out, nil } +// versionCapture is what the §§version§§ placeholder in a Retire.js pattern +// stands for: one capture group around a dotted numeric version. +const versionCapture = `([0-9]+(?:\.[0-9]+)+)` + func compileAll(name, extractor string, patterns []string) ([]*regexp2.Regexp, error) { var out []*regexp2.Regexp for _, p := range patterns { + p = strings.ReplaceAll(p, "§§version§§", versionCapture) re, err := regexp2.Compile(p, regexp2.ECMAScript) if err != nil { return nil, fmt.Errorf("fingerprint: %s %s pattern %q: %w", name, extractor, p, err) From 4346d277e8ca3bee2e609618b0a98ee68e4eb931 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:04:03 +0000 Subject: [PATCH 4/4] fix(fingerprint): expand version placeholder in Retire.js patterns (#15) Co-authored-by: hsinatfootprintai <149764663+hsinatfootprintai@users.noreply.github.com> --- internal/fingerprint/fingerprint.go | 14 +++++++----- internal/fingerprint/fingerprint_test.go | 29 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index b8e3e20..36c87d2 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -28,6 +28,13 @@ import ( // and the URLs they run against come from attacker-controlled pages. const matchTimeout = 100 * time.Millisecond +// versionPlaceholder marks the version capture in Retire.js patterns; +// compileAll replaces it with versionCapture, a dotted numeric version. +const ( + versionPlaceholder = "§§version§§" + versionCapture = `([0-9]+(?:\.[0-9]+)+)` +) + //go:embed data/signatures.json var signatureData []byte @@ -94,15 +101,10 @@ func compile(data []byte) ([]library, error) { return out, nil } -// versionCapture is what the §§version§§ placeholder in a Retire.js pattern -// stands for: one capture group around a dotted numeric version. -const versionCapture = `([0-9]+(?:\.[0-9]+)+)` - func compileAll(name, extractor string, patterns []string) ([]*regexp2.Regexp, error) { var out []*regexp2.Regexp for _, p := range patterns { - p = strings.ReplaceAll(p, "§§version§§", versionCapture) - re, err := regexp2.Compile(p, regexp2.ECMAScript) + re, err := regexp2.Compile(strings.ReplaceAll(p, versionPlaceholder, versionCapture), regexp2.ECMAScript) if err != nil { return nil, fmt.Errorf("fingerprint: %s %s pattern %q: %w", name, extractor, p, err) } diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 6ec00d6..ca23684 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -181,6 +181,35 @@ func TestFromContent(t *testing.T) { } } +// The embedded React and Vue patterns use the §§version§§ placeholder, which +// compileAll must turn into a version capture for them to match at all. +func TestFromContentReactVue(t *testing.T) { + const asset = "https://vendor.example/static/bundle.js" + tests := []struct { + name string + body string + want string // "name version" + }{ + {"react license banner", "/** @license React v16.14.0\n * react.production.min.js\n */\n!function(){}();", "react 16.14.0"}, + {"vue banner", "/*!\n * Vue.js v2.6.14\n * (c) 2014-2021 Evan You\n */\nvar x = 1;", "vue 2.6.14"}, + {"vue version assignment", "var Vue = {}; Vue.version = '3.2.47';", "vue 3.2.47"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := matchContent(libraries, asset, []byte(tt.body)) + if err != nil { + t.Fatalf("matchContent: %v", err) + } + if len(got) != 1 { + t.Fatalf("matchContent = %+v, want one component", got) + } + if c := got[0]; c.Name+" "+c.Version != tt.want { + t.Errorf("component = %q %q; want %q", c.Name, c.Version, tt.want) + } + }) + } +} + // A catastrophic pattern must be stopped by MatchTimeout and reported as an // error, not left to run for the length of the input. func TestMatchTimeout(t *testing.T) {