diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7d..3e3290a691 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -8,6 +8,7 @@ The following organizations or individuals have contributed to ScanCode: - Akanksha Garg @akugarg - Alex Blekhman @a-tinsmith - Alexander Gschrei @agschrei +- Arbaz Khan @arbazkhan971 - Armijn Hemmel @armijnhemel - Armin Tänzer @armintaenzertng - Arnaud Jeansen @ajeans diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf4db41092..59a68b131a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Changelog Next release -------------- +- Add support for parsing ``go mod graph`` dumps as ``go-mod-graph.deplock`` + and ``go.mod.graph`` package datafiles. + https://github.com/aboutcode-org/scancode-toolkit/issues/4423 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/docs/source/reference/scancode-supported-packages.rst b/docs/source/reference/scancode-supported-packages.rst index d2ad15803f..7c71c16679 100644 --- a/docs/source/reference/scancode-supported-packages.rst +++ b/docs/source/reference/scancode-supported-packages.rst @@ -491,6 +491,13 @@ parsers in scancode-toolkit during documentation builds. - ``go_mod`` - Go - https://go.dev/ref/mod + * - Go module requirement graph + - ``*/go-mod-graph.deplock``, ``*/go.mod.graph`` + - ``golang`` + - ``linux``, ``win``, ``mac`` + - ``go_mod_graph`` + - Go + - https://go.dev/ref/mod#go-mod-graph * - Go module cheksums file - ``*/go.sum`` - ``golang`` diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index fc1e490eef..cd6c2f102f 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -102,6 +102,7 @@ godeps.GodepsHandler, golang.GoModHandler, golang.GoSumHandler, + golang.GoModGraphHandler, haxe.HaxelibJsonHandler, diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 527931c2dc..9de30e30a7 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -253,3 +253,68 @@ def parse_gosum(location): gosums.append(dep) return gosums + + +def split_module_version(token): + """ + Return a GoModule parsed from a ``go mod graph`` token. + + Each token is a module path, optionally followed by ``@`` and a version. + The main module is typically printed without a version. + + For example:: + + >>> m = split_module_version('example.com/my/thing') + >>> assert m.namespace == 'example.com/my' + >>> assert m.name == 'thing' + >>> assert m.version is None + >>> assert m.module == 'example.com/my/thing' + + >>> m = split_module_version('github.com/davecgh/go-spew@v1.1.1') + >>> assert m.namespace == 'github.com/davecgh' + >>> assert m.name == 'go-spew' + >>> assert m.version == 'v1.1.1' + >>> assert m.module == 'github.com/davecgh/go-spew' + """ + if '@' in token: + path, version = token.rsplit('@', 1) + else: + path, version = token, None + namespace, _, name = path.rpartition('/') + return GoModule( + namespace=namespace or None, + name=name, + version=version, + module=path, + ) + + +def parse_gograph(location): + """ + Return a list of (requiring, required) GoModule pairs from a ``go mod graph`` + dump at ``location``. + + See https://go.dev/ref/mod#go-mod-graph + + Each line is two space-separated module versions: the requiring module, + then the required module. + + For example:: + + example.com/main example.com/m1@v1.0.0 + example.com/m1@v1.0.0 example.com/m2@v1.1.0 + """ + edges = [] + with io.open(location, encoding='utf-8', closefd=True) as data: + for raw_line in data: + line = raw_line.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) != 2: + continue + edges.append(( + split_module_version(parts[0]), + split_module_version(parts[1]), + )) + return edges diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index c45be7a5e3..38571afa53 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -33,7 +33,12 @@ def assemble(cls, package_data, resource, codebase, package_adder): Always use go.mod first then go.sum """ yield from cls.assemble_from_many_datafiles( - datafile_name_patterns=('go.mod', 'go.sum',), + datafile_name_patterns=( + 'go.mod', + 'go.sum', + 'go-mod-graph.deplock', + 'go.mod.graph', + ), directory=resource.parent(codebase), codebase=codebase, package_adder=package_adder, @@ -134,3 +139,75 @@ def parse(cls, location, package_only=False): primary_language=cls.default_primary_language, ) yield models.PackageData.from_data(package_data, package_only) + + +class GoModGraphHandler(BaseGoModuleHandler): + datasource_id = 'go_mod_graph' + path_patterns = ('*/go-mod-graph.deplock', '*/go.mod.graph') + default_package_type = 'golang' + default_primary_language = 'Go' + description = 'Go module requirement graph from go mod graph' + documentation_url = 'https://go.dev/ref/mod#go-mod-graph' + + @classmethod + def parse(cls, location, package_only=False): + """ + Parse a ``go mod graph`` dump. + + Direct dependencies are modules required by the main module (the first + requiring module in the dump). Other required modules are transitive. + Versions in the graph are exact selected versions. + """ + edges = go_mod.parse_gograph(location) + if not edges: + return + + main = edges[0][0] + direct_purls = { + dst.purl(include_version=True) + for src, dst in edges + if src.module == main.module + } + + dependencies = [] + seen = set() + for _src, dst in edges: + purl = dst.purl(include_version=True) + if purl in seen: + continue + seen.add(purl) + dependencies.append( + models.DependentPackage( + purl=purl, + extracted_requirement=dst.version, + scope='require', + is_runtime=True, + is_optional=False, + is_pinned=True, + is_direct=purl in direct_purls, + ) + ) + + namespace = main.namespace + name = main.name + homepage_url = None + vcs_url = None + repository_homepage_url = None + if namespace and name: + homepage_url = f'https://pkg.go.dev/{namespace}/{name}' + vcs_url = f'https://{namespace}/{name}.git' + repository_homepage_url = homepage_url + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name=name, + namespace=namespace, + version=main.version, + vcs_url=vcs_url, + homepage_url=homepage_url, + repository_homepage_url=repository_homepage_url, + dependencies=dependencies, + primary_language=cls.default_primary_language, + ) + yield models.PackageData.from_data(package_data, package_only) diff --git a/tests/packagedcode/data/golang/gograph/sample/go-mod-graph.deplock b/tests/packagedcode/data/golang/gograph/sample/go-mod-graph.deplock new file mode 100644 index 0000000000..462cc9187e --- /dev/null +++ b/tests/packagedcode/data/golang/gograph/sample/go-mod-graph.deplock @@ -0,0 +1,4 @@ +example.com/my/thing example.com/other/thing@v1.0.2 +example.com/my/thing example.com/new/thing@v2.3.4 +example.com/other/thing@v1.0.2 golang.org/x/text@v0.3.0 +example.com/new/thing@v2.3.4 github.com/davecgh/go-spew@v1.1.1 diff --git a/tests/packagedcode/data/golang/gograph/sample/output.expected.json b/tests/packagedcode/data/golang/gograph/sample/output.expected.json new file mode 100644 index 0000000000..4795ce8993 --- /dev/null +++ b/tests/packagedcode/data/golang/gograph/sample/output.expected.json @@ -0,0 +1,91 @@ +[ + { + "type": "golang", + "namespace": "example.com/my", + "name": "thing", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://pkg.go.dev/example.com/my/thing", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://example.com/my/thing.git", + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:golang/example.com/other/thing@v1.0.2", + "extracted_requirement": "v1.0.2", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/example.com/new/thing@v2.3.4", + "extracted_requirement": "v2.3.4", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/text@v0.3.0", + "extracted_requirement": "v0.3.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/davecgh/go-spew@v1.1.1", + "extracted_requirement": "v1.1.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pkg.go.dev/example.com/my/thing", + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "go_mod_graph", + "purl": "pkg:golang/example.com/my/thing" + } +] diff --git a/tests/packagedcode/data/plugin/help.txt b/tests/packagedcode/data/plugin/help.txt index 7128425a96..588317e0cb 100755 --- a/tests/packagedcode/data/plugin/help.txt +++ b/tests/packagedcode/data/plugin/help.txt @@ -433,6 +433,13 @@ Package type: golang description: Go modules file path_patterns: '*/go.mod' -------------------------------------------- +Package type: golang + datasource_id: go_mod_graph + documentation URL: https://go.dev/ref/mod#go-mod-graph + primary language: Go + description: Go module requirement graph from go mod graph + path_patterns: '*/go-mod-graph.deplock', '*/go.mod.graph' +-------------------------------------------- Package type: golang datasource_id: go_sum documentation URL: https://go.dev/ref/mod#go-sum-files diff --git a/tests/packagedcode/data/plugin/plugins_list_linux.txt b/tests/packagedcode/data/plugin/plugins_list_linux.txt index 34078bc580..e20257f29a 100755 --- a/tests/packagedcode/data/plugin/plugins_list_linux.txt +++ b/tests/packagedcode/data/plugin/plugins_list_linux.txt @@ -454,6 +454,13 @@ Package type: golang description: Go modules file path_patterns: '*/go.mod' -------------------------------------------- +Package type: golang + datasource_id: go_mod_graph + documentation URL: https://go.dev/ref/mod#go-mod-graph + primary language: Go + description: Go module requirement graph from go mod graph + path_patterns: '*/go-mod-graph.deplock', '*/go.mod.graph' +-------------------------------------------- Package type: golang datasource_id: go_sum documentation URL: https://go.dev/ref/mod#go-sum-files diff --git a/tests/packagedcode/test_golang.py b/tests/packagedcode/test_golang.py index 58feea2def..a1c987d1b4 100644 --- a/tests/packagedcode/test_golang.py +++ b/tests/packagedcode/test_golang.py @@ -99,3 +99,17 @@ def test_parse_gosum_sample6(self): expected_loc = self.get_test_loc('golang/gosum/sample6/output.expected.json') package = golang.GoSumHandler.parse(test_file) self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_gograph_is_package_data_file(self): + test_file = self.get_test_loc('golang/gograph/sample/go-mod-graph.deplock') + assert golang.GoModGraphHandler.is_datafile(test_file) + assert golang.GoModGraphHandler.is_datafile( + 'project/go.mod.graph', + _bare_filename=True, + ) + + def test_parse_gograph_sample(self): + test_file = self.get_test_loc('golang/gograph/sample/go-mod-graph.deplock') + expected_loc = self.get_test_loc('golang/gograph/sample/output.expected.json') + package = golang.GoModGraphHandler.parse(test_file) + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) diff --git a/tests/packagedcode/test_recognize.py b/tests/packagedcode/test_recognize.py index 98a5016432..5db220b22f 100644 --- a/tests/packagedcode/test_recognize.py +++ b/tests/packagedcode/test_recognize.py @@ -197,6 +197,12 @@ def test_recognize_go_sum(self): assert packages assert isinstance(packages[0], models.PackageData) + def test_recognize_go_mod_graph(self): + test_file = self.get_test_loc('golang/gograph/sample/go-mod-graph.deplock') + packages = recognize_package_data(test_file) + assert packages + assert isinstance(packages[0], models.PackageData) + def test_recognize_rpmdb_sqlite(self): test_file = self.get_test_loc('rpm/rpmdb.sqlite') packages = recognize_package_data(test_file, system=True)