diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..3bb3d61 --- /dev/null +++ b/NOTICE @@ -0,0 +1,7 @@ +Touchstone +Copyright FootprintAI + +This product includes signature data derived from Retire.js +(https://github.com/RetireJS/retire.js), jsrepository-v4.json, +licensed under the Apache License, Version 2.0 +(http://www.apache.org/licenses/LICENSE-2.0). diff --git a/internal/fingerprint/data/signatures.json b/internal/fingerprint/data/signatures.json index 571c088..d82d691 100644 --- a/internal/fingerprint/data/signatures.json +++ b/internal/fingerprint/data/signatures.json @@ -1,5 +1,5 @@ { - "_comment": "Retire.js-style signatures, uri and filename extractors only. Hand-curated for slice S2 (jQuery only); to be replaced by a pinned snapshot of Retire.js jsrepository in S7. {version} in purl and cpe is replaced by the captured version.", + "_comment": "Retire.js-style signatures (uri, filename, filecontent, hashes). Hand-curated subset in the Retire.js jsrepository shape; to be replaced by a pinned snapshot of jsrepository-v4.json (Apache-2.0, see NOTICE). {version} in purl and cpe is replaced by the captured version; §§version§§ in a filecontent pattern is the version capture.", "libraries": [ { "name": "jquery", @@ -9,6 +9,27 @@ "uri": ["jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js"], "filename": ["^jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js$"] } + }, + { + "name": "react", + "purl": "pkg:npm/react@{version}", + "cpe": "cpe:2.3:a:facebook:react:{version}:*:*:*:*:*:*:*", + "extractors": { + "filecontent": [ + "/\\*\\*?\\s*@license React v§§version§§" + ] + } + }, + { + "name": "vue", + "purl": "pkg:npm/vue@{version}", + "cpe": "cpe:2.3:a:vuejs:vue:{version}:*:*:*:*:*:*:*", + "extractors": { + "filecontent": [ + "/\\*!\\s*\\*\\s*Vue\\.js v§§version§§", + "Vue\\.version\\s*=\\s*[\"']§§version§§[\"']" + ] + } } ] } diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index 5aaedd6..36c87d2 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -6,7 +6,9 @@ package fingerprint import ( "bytes" + "crypto/sha1" _ "embed" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -26,6 +28,13 @@ import ( // and the URLs they run against come from attacker-controlled pages. const matchTimeout = 100 * time.Millisecond +// versionPlaceholder marks the version capture in Retire.js patterns; +// compileAll replaces it with versionCapture, a dotted numeric version. +const ( + versionPlaceholder = "§§version§§" + versionCapture = `([0-9]+(?:\.[0-9]+)+)` +) + //go:embed data/signatures.json var signatureData []byte @@ -36,16 +45,21 @@ type signatureFile struct { Purl string `json:"purl"` CPE string `json:"cpe"` Extractors struct { - URI []string `json:"uri"` - Filename []string `json:"filename"` + URI []string `json:"uri"` + Filename []string `json:"filename"` + FileContent []string `json:"filecontent"` + Hashes map[string]string `json:"hashes"` } `json:"extractors"` } `json:"libraries"` } -// library is one compiled signature entry. +// library is one compiled signature entry. hashes maps the lower-case hex +// SHA-1 of a whole asset to the version it is. type library struct { name, purl, cpe string uri, filename []*regexp2.Regexp + filecontent []*regexp2.Regexp + hashes map[string]string } // libraries is compiled once at start-up from the embedded signatures. A @@ -68,7 +82,7 @@ func compile(data []byte) ([]library, error) { } var out []library for _, l := range sf.Libraries { - lib := library{name: l.Name, purl: l.Purl, cpe: l.CPE} + lib := library{name: l.Name, purl: l.Purl, cpe: l.CPE, hashes: map[string]string{}} var err error if lib.uri, err = compileAll(l.Name, "uri", l.Extractors.URI); err != nil { return nil, err @@ -76,6 +90,12 @@ func compile(data []byte) ([]library, error) { if lib.filename, err = compileAll(l.Name, "filename", l.Extractors.Filename); err != nil { return nil, err } + if lib.filecontent, err = compileAll(l.Name, "filecontent", l.Extractors.FileContent); err != nil { + return nil, err + } + for digest, version := range l.Extractors.Hashes { + lib.hashes[strings.ToLower(digest)] = version + } out = append(out, lib) } return out, nil @@ -84,7 +104,7 @@ func compile(data []byte) ([]library, error) { func compileAll(name, extractor string, patterns []string) ([]*regexp2.Regexp, error) { var out []*regexp2.Regexp for _, p := range patterns { - re, err := regexp2.Compile(p, regexp2.ECMAScript) + re, err := regexp2.Compile(strings.ReplaceAll(p, versionPlaceholder, versionCapture), regexp2.ECMAScript) if err != nil { return nil, fmt.Errorf("fingerprint: %s %s pattern %q: %w", name, extractor, p, err) } @@ -131,6 +151,45 @@ func FromURL(assetURL string) ([]pipeline.Component, error) { return out, errors.Join(errs...) } +// FromContent matches the bytes of a script that FromURL did not recognise: +// the filecontent extractors run over the text, and the hashes extractors +// over the SHA-1 of the whole asset. Each library it recognises yields one +// Low-confidence component, with the asset URL as evidence. +func FromContent(assetURL string, body []byte) ([]pipeline.Component, error) { + return matchContent(libraries, assetURL, body) +} + +func matchContent(libs []library, assetURL string, body []byte) ([]pipeline.Component, error) { + sum := sha1.Sum(body) + digest := hex.EncodeToString(sum[:]) + text := string(body) + var out []pipeline.Component + var errs []error + for _, lib := range libs { + if version, ok := lib.hashes[digest]; ok { + c := lib.component(version) + c.Evidence = []pipeline.Evidence{{Kind: "FINGERPRINT_ASSET", Location: assetURL, Detail: "sha1 " + digest}} + out = append(out, c) + continue + } + c, ok, err := lib.match(text, lib.filecontent) + if err != nil { + errs = append(errs, err) + continue + } + if !ok { + continue + } + c.Evidence = []pipeline.Evidence{{ + Kind: "FINGERPRINT_ASSET", + Location: assetURL, + Detail: c.Evidence[0].Detail, + }} + out = append(out, c) + } + return out, errors.Join(errs...) +} + // match runs res against input and, on the first hit, builds a component // with the captured version. The evidence detail is the matched text. func (l library) match(input string, res []*regexp2.Regexp) (pipeline.Component, bool, error) { @@ -142,20 +201,25 @@ func (l library) match(input string, res []*regexp2.Regexp) (pipeline.Component, if m == nil { continue } - version := m.GroupByNumber(1).String() - c := pipeline.Component{ - Name: l.name, - Version: version, - Purl: strings.ReplaceAll(l.purl, "{version}", version), - CPE: strings.ReplaceAll(l.cpe, "{version}", version), - Confidence: "LOW", - Evidence: []pipeline.Evidence{{Detail: m.String()}}, - } + c := l.component(m.GroupByNumber(1).String()) + c.Evidence = []pipeline.Evidence{{Detail: m.String()}} return c, true, nil } return pipeline.Component{}, false, nil } +// component builds the library's component at version, with its purl and +// CPE filled in. The caller adds the evidence. +func (l library) component(version string) pipeline.Component { + return pipeline.Component{ + Name: l.name, + Version: version, + Purl: strings.ReplaceAll(l.purl, "{version}", version), + CPE: strings.ReplaceAll(l.cpe, "{version}", version), + Confidence: "LOW", + } +} + // headerRule maps one response header value to a component. Banners // describe the server software, which distributions may patch without // changing the version string, hence the MAY_BE_BACKPORTED caveat. @@ -169,6 +233,7 @@ type headerRule struct { var headerRules = []headerRule{ {"Server", regexp.MustCompile(`^Apache/([0-9]+(?:\.[0-9]+)+)`), "apache http_server", "cpe:2.3:a:apache:http_server:%s:*:*:*:*:*:*:*"}, {"Server", regexp.MustCompile(`^nginx/([0-9]+(?:\.[0-9]+)+)`), "nginx", "cpe:2.3:a:f5:nginx:%s:*:*:*:*:*:*:*"}, + {"Server", regexp.MustCompile(`^Microsoft-IIS/([0-9]+(?:\.[0-9]+)+)`), "iis", "cpe:2.3:a:microsoft:internet_information_services:%s:*:*:*:*:*:*:*"}, {"X-Powered-By", regexp.MustCompile(`^PHP/([0-9]+(?:\.[0-9]+)+)`), "php", "cpe:2.3:a:php:php:%s:*:*:*:*:*:*:*"}, } diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 4e2d13d..ca23684 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" "testing" + "time" ) func TestSignaturesCompile(t *testing.T) { @@ -98,6 +99,7 @@ func TestFromHeaders(t *testing.T) { }{ {"apache", http.Header{"Server": {"Apache/2.4.29 (Ubuntu)"}}, []string{"apache http_server 2.4.29"}}, {"nginx", http.Header{"Server": {"nginx/1.18.0"}}, []string{"nginx 1.18.0"}}, + {"iis", http.Header{"Server": {"Microsoft-IIS/10.0"}}, []string{"iis 10.0"}}, {"php", http.Header{"X-Powered-By": {"PHP/7.2.34"}}, []string{"php 7.2.34"}}, {"unknown banner", http.Header{"Server": {"cloudflare"}}, nil}, {"no headers", http.Header{}, nil}, @@ -128,6 +130,104 @@ func TestFromHeaders(t *testing.T) { } } +// testSignatures has one library with a content pattern and one with a +// SHA-1 entry. sha1("abc") is a9993e36 4706816a ba3e2571 7850c26c 9cd0d89d. +const testSignatures = `{"libraries":[ + {"name":"fakelib","purl":"pkg:npm/fakelib@{version}","cpe":"cpe:2.3:a:fake:fakelib:{version}:*:*:*:*:*:*:*", + "extractors":{"filecontent":["/\\*! FakeLib v([0-9]+\\.[0-9]+\\.[0-9]+)"]}}, + {"name":"hashed","purl":"pkg:npm/hashed@{version}","cpe":"","extractors":{"hashes":{"A9993E364706816ABA3E25717850C26C9CD0D89D":"1.0.0"}}} +]}` + +func TestFromContent(t *testing.T) { + libs, err := compile([]byte(testSignatures)) + if err != nil { + t.Fatalf("compile: %v", err) + } + const asset = "https://vendor.example/static/bundle.js" + tests := []struct { + name string + body string + want string // "name version"; "" = no component + detail string + }{ + {"banner in content", "/*! FakeLib v2.3.4 */ var x = 1;", "fakelib 2.3.4", "/*! FakeLib v2.3.4"}, + {"sha1 of whole asset", "abc", "hashed 1.0.0", "sha1 a9993e364706816aba3e25717850c26c9cd0d89d"}, + {"no match", "var x = 1;", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := matchContent(libs, asset, []byte(tt.body)) + if err != nil { + t.Fatalf("matchContent: %v", err) + } + if tt.want == "" { + if len(got) != 0 { + t.Fatalf("matchContent = %+v, want no components", got) + } + return + } + if len(got) != 1 { + t.Fatalf("matchContent = %+v, want one component", got) + } + c := got[0] + if c.Name+" "+c.Version != tt.want || c.Confidence != "LOW" { + t.Errorf("component = %q %q confidence %q; want %q LOW", c.Name, c.Version, c.Confidence, tt.want) + } + if len(c.Evidence) != 1 || c.Evidence[0].Kind != "FINGERPRINT_ASSET" || + c.Evidence[0].Location != asset || c.Evidence[0].Detail != tt.detail { + t.Errorf("evidence = %+v, want asset evidence at %s detail %q", c.Evidence, asset, tt.detail) + } + }) + } +} + +// The embedded React and Vue patterns use the §§version§§ placeholder, which +// compileAll must turn into a version capture for them to match at all. +func TestFromContentReactVue(t *testing.T) { + const asset = "https://vendor.example/static/bundle.js" + tests := []struct { + name string + body string + want string // "name version" + }{ + {"react license banner", "/** @license React v16.14.0\n * react.production.min.js\n */\n!function(){}();", "react 16.14.0"}, + {"vue banner", "/*!\n * Vue.js v2.6.14\n * (c) 2014-2021 Evan You\n */\nvar x = 1;", "vue 2.6.14"}, + {"vue version assignment", "var Vue = {}; Vue.version = '3.2.47';", "vue 3.2.47"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := matchContent(libraries, asset, []byte(tt.body)) + if err != nil { + t.Fatalf("matchContent: %v", err) + } + if len(got) != 1 { + t.Fatalf("matchContent = %+v, want one component", got) + } + if c := got[0]; c.Name+" "+c.Version != tt.want { + t.Errorf("component = %q %q; want %q", c.Name, c.Version, tt.want) + } + }) + } +} + +// A catastrophic pattern must be stopped by MatchTimeout and reported as an +// error, not left to run for the length of the input. +func TestMatchTimeout(t *testing.T) { + libs, err := compile([]byte(`{"libraries":[{"name":"redos","purl":"","cpe":"","extractors":{"filecontent":["^(a+)+$"]}}]}`)) + if err != nil { + t.Fatalf("compile: %v", err) + } + body := []byte(strings.Repeat("a", 40) + "!") + start := time.Now() + _, err = matchContent(libs, "https://vendor.example/x.js", body) + if err == nil { + t.Fatal("matchContent returned no error; want a timeout") + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("matchContent took %v; want it stopped near the 100ms timeout", elapsed) + } +} + func contains(list []string, want string) bool { for _, s := range list { if s == want { diff --git a/internal/service/scanner.go b/internal/service/scanner.go index 76c139e..6607cea 100644 --- a/internal/service/scanner.go +++ b/internal/service/scanner.go @@ -110,10 +110,18 @@ func collect(ctx context.Context, f fetch.Fetcher, page *fetch.Response) ([]pipe continue } found, err := fingerprint.FromURL(src) - comps = append(comps, found...) if err != nil { errs = append(errs, err) } + if len(found) == 0 { + // Content is matched only when the URL names no library, so one + // script yields at most one match per library. + found, err = fingerprint.FromContent(src, resp.Body) + if err != nil { + errs = append(errs, err) + } + } + comps = append(comps, found...) } return merge(comps), errors.Join(errs...) }