Add support for parsing go mod graph dumps

AUTHORS.rst

@@ -8,6 +8,7 @@ The following organizations or individuals have contributed to ScanCode:
 - Akanksha Garg @akugarg
 - Alex Blekhman @a-tinsmith
 - Alexander Gschrei @agschrei
+- Arbaz Khan @arbazkhan971
 - Armijn Hemmel @armijnhemel
 - Armin Tänzer @armintaenzertng
 - Arnaud Jeansen @ajeans

CHANGELOG.rst

@@ -4,6 +4,10 @@ Changelog
 Next release
 --------------
 
+- Add support for parsing ``go mod graph`` dumps as ``go-mod-graph.deplock``
+  and ``go.mod.graph`` package datafiles.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/4423
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

docs/source/reference/scancode-supported-packages.rst

@@ -491,6 +491,13 @@ parsers in scancode-toolkit during documentation builds.
      - ``go_mod``
      - Go
      - https://go.dev/ref/mod
+   * - Go module requirement graph
+     - ``*/go-mod-graph.deplock``, ``*/go.mod.graph``
+     - ``golang``
+     - ``linux``, ``win``, ``mac``
+     - ``go_mod_graph``
+     - Go
+     - https://go.dev/ref/mod#go-mod-graph
    * - Go module cheksums file
      - ``*/go.sum``
      - ``golang``

src/packagedcode/__init__.py

@@ -102,6 +102,7 @@
     godeps.GodepsHandler,
     golang.GoModHandler,
     golang.GoSumHandler,
+    golang.GoModGraphHandler,
 
     haxe.HaxelibJsonHandler,
 

src/packagedcode/go_mod.py

@@ -253,3 +253,68 @@ def parse_gosum(location):
         gosums.append(dep)
 
     return gosums
+
+
+def split_module_version(token):
+    """
+    Return a GoModule parsed from a ``go mod graph`` token.
+
+    Each token is a module path, optionally followed by ``@`` and a version.
+    The main module is typically printed without a version.
+
+    For example::
+
+        >>> m = split_module_version('example.com/my/thing')
+        >>> assert m.namespace == 'example.com/my'
+        >>> assert m.name == 'thing'
+        >>> assert m.version is None
+        >>> assert m.module == 'example.com/my/thing'
+
+        >>> m = split_module_version('github.com/davecgh/go-spew@v1.1.1')
+        >>> assert m.namespace == 'github.com/davecgh'
+        >>> assert m.name == 'go-spew'
+        >>> assert m.version == 'v1.1.1'
+        >>> assert m.module == 'github.com/davecgh/go-spew'
+    """
+    if '@' in token:
+        path, version = token.rsplit('@', 1)
+    else:
+        path, version = token, None
+    namespace, _, name = path.rpartition('/')
+    return GoModule(
+        namespace=namespace or None,
+        name=name,
+        version=version,
+        module=path,
+    )
+
+
+def parse_gograph(location):
+    """
+    Return a list of (requiring, required) GoModule pairs from a ``go mod graph``
+    dump at ``location``.
+
+    See https://go.dev/ref/mod#go-mod-graph
+
+    Each line is two space-separated module versions: the requiring module,
+    then the required module.
+
+    For example::
+
+        example.com/main example.com/m1@v1.0.0
+        example.com/m1@v1.0.0 example.com/m2@v1.1.0
+    """
+    edges = []
+    with io.open(location, encoding='utf-8', closefd=True) as data:
+        for raw_line in data:
+            line = raw_line.strip()
+            if not line or line.startswith('#'):
+                continue
+            parts = line.split()
+            if len(parts) != 2:
+                continue
+            edges.append((
+                split_module_version(parts[0]),
+                split_module_version(parts[1]),
+            ))
+    return edges

src/packagedcode/golang.py

@@ -33,7 +33,12 @@ def assemble(cls, package_data, resource, codebase, package_adder):
         Always use go.mod first then go.sum
         """
         yield from cls.assemble_from_many_datafiles(
-            datafile_name_patterns=('go.mod', 'go.sum',),
+            datafile_name_patterns=(
+                'go.mod',
+                'go.sum',
+                'go-mod-graph.deplock',
+                'go.mod.graph',
+            ),
             directory=resource.parent(codebase),
             codebase=codebase,
             package_adder=package_adder,
@@ -134,3 +139,75 @@ def parse(cls, location, package_only=False):
             primary_language=cls.default_primary_language,
         )
         yield models.PackageData.from_data(package_data, package_only)
+
+
+class GoModGraphHandler(BaseGoModuleHandler):
+    datasource_id = 'go_mod_graph'
+    path_patterns = ('*/go-mod-graph.deplock', '*/go.mod.graph')
+    default_package_type = 'golang'
+    default_primary_language = 'Go'
+    description = 'Go module requirement graph from go mod graph'
+    documentation_url = 'https://go.dev/ref/mod#go-mod-graph'
+
+    @classmethod
+    def parse(cls, location, package_only=False):
+        """
+        Parse a ``go mod graph`` dump.
+
+        Direct dependencies are modules required by the main module (the first
+        requiring module in the dump). Other required modules are transitive.
+        Versions in the graph are exact selected versions.
+        """
+        edges = go_mod.parse_gograph(location)
+        if not edges:
+            return
+
+        main = edges[0][0]
+        direct_purls = {
+            dst.purl(include_version=True)
+            for src, dst in edges
+            if src.module == main.module
+        }
+
+        dependencies = []
+        seen = set()
+        for _src, dst in edges:
+            purl = dst.purl(include_version=True)
+            if purl in seen:
+                continue
+            seen.add(purl)
+            dependencies.append(
+                models.DependentPackage(
+                    purl=purl,
+                    extracted_requirement=dst.version,
+                    scope='require',
+                    is_runtime=True,
+                    is_optional=False,
+                    is_pinned=True,
+                    is_direct=purl in direct_purls,
+                )
+            )
+
+        namespace = main.namespace
+        name = main.name
+        homepage_url = None
+        vcs_url = None
+        repository_homepage_url = None
+        if namespace and name:
+            homepage_url = f'https://pkg.go.dev/{namespace}/{name}'
+            vcs_url = f'https://{namespace}/{name}.git'
+            repository_homepage_url = homepage_url
+
+        package_data = dict(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            name=name,
+            namespace=namespace,
+            version=main.version,
+            vcs_url=vcs_url,
+            homepage_url=homepage_url,
+            repository_homepage_url=repository_homepage_url,
+            dependencies=dependencies,
+            primary_language=cls.default_primary_language,
+        )
+        yield models.PackageData.from_data(package_data, package_only)

tests/packagedcode/data/golang/gograph/sample/go-mod-graph.deplock

@@ -0,0 +1,4 @@
+example.com/my/thing example.com/other/thing@v1.0.2
+example.com/my/thing example.com/new/thing@v2.3.4
+example.com/other/thing@v1.0.2 golang.org/x/text@v0.3.0
+example.com/new/thing@v2.3.4 github.com/davecgh/go-spew@v1.1.1

tests/packagedcode/data/golang/gograph/sample/output.expected.json

@@ -0,0 +1,91 @@
+[
+  {
+    "type": "golang",
+    "namespace": "example.com/my",
+    "name": "thing",
+    "version": null,
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": "Go",
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": "https://pkg.go.dev/example.com/my/thing",
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": "https://example.com/my/thing.git",
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [
+      {
+        "purl": "pkg:golang/example.com/other/thing@v1.0.2",
+        "extracted_requirement": "v1.0.2",
+        "scope": "require",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": true,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:golang/example.com/new/thing@v2.3.4",
+        "extracted_requirement": "v2.3.4",
+        "scope": "require",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": true,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:golang/golang.org/x/text@v0.3.0",
+        "extracted_requirement": "v0.3.0",
+        "scope": "require",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": true,
+        "is_direct": false,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:golang/github.com/davecgh/go-spew@v1.1.1",
+        "extracted_requirement": "v1.1.1",
+        "scope": "require",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": true,
+        "is_direct": false,
+        "resolved_package": {},
+        "extra_data": {}
+      }
+    ],
+    "repository_homepage_url": "https://pkg.go.dev/example.com/my/thing",
+    "repository_download_url": null,
+    "api_data_url": null,
+    "datasource_id": "go_mod_graph",
+    "purl": "pkg:golang/example.com/my/thing"
+  }
+]

tests/packagedcode/data/plugin/help.txt

@@ -433,6 +433,13 @@ Package type:  golang
   description:       Go modules file
   path_patterns:    '*/go.mod'
 --------------------------------------------
+Package type:  golang
+  datasource_id:     go_mod_graph
+  documentation URL: https://go.dev/ref/mod#go-mod-graph
+  primary language:  Go
+  description:       Go module requirement graph from go mod graph
+  path_patterns:    '*/go-mod-graph.deplock', '*/go.mod.graph'
+--------------------------------------------
 Package type:  golang
   datasource_id:     go_sum
   documentation URL: https://go.dev/ref/mod#go-sum-files

tests/packagedcode/data/plugin/plugins_list_linux.txt

@@ -454,6 +454,13 @@ Package type:  golang
   description:       Go modules file
   path_patterns:    '*/go.mod'
 --------------------------------------------
+Package type:  golang
+  datasource_id:     go_mod_graph
+  documentation URL: https://go.dev/ref/mod#go-mod-graph
+  primary language:  Go
+  description:       Go module requirement graph from go mod graph
+  path_patterns:    '*/go-mod-graph.deplock', '*/go.mod.graph'
+--------------------------------------------
 Package type:  golang
   datasource_id:     go_sum
   documentation URL: https://go.dev/ref/mod#go-sum-files

tests/packagedcode/test_golang.py

@@ -99,3 +99,17 @@ def test_parse_gosum_sample6(self):
         expected_loc = self.get_test_loc('golang/gosum/sample6/output.expected.json')
         package = golang.GoSumHandler.parse(test_file)
         self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
+
+    def test_gograph_is_package_data_file(self):
+        test_file = self.get_test_loc('golang/gograph/sample/go-mod-graph.deplock')
+        assert golang.GoModGraphHandler.is_datafile(test_file)
+        assert golang.GoModGraphHandler.is_datafile(
+            'project/go.mod.graph',
+            _bare_filename=True,
+        )
+
+    def test_parse_gograph_sample(self):
+        test_file = self.get_test_loc('golang/gograph/sample/go-mod-graph.deplock')
+        expected_loc = self.get_test_loc('golang/gograph/sample/output.expected.json')
+        package = golang.GoModGraphHandler.parse(test_file)
+        self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)

tests/packagedcode/test_recognize.py

@@ -197,6 +197,12 @@ def test_recognize_go_sum(self):
         assert packages
         assert isinstance(packages[0], models.PackageData)
 
+    def test_recognize_go_mod_graph(self):
+        test_file = self.get_test_loc('golang/gograph/sample/go-mod-graph.deplock')
+        packages = recognize_package_data(test_file)
+        assert packages
+        assert isinstance(packages[0], models.PackageData)
+
     def test_recognize_rpmdb_sqlite(self):
         test_file = self.get_test_loc('rpm/rpmdb.sqlite')
         packages = recognize_package_data(test_file, system=True)