feat(parser): mark dev-dependencies component.scope as 'excluded'

cyclonedx_py/_internal/pipenv.py

@@ -26,7 +26,7 @@
 
 from cyclonedx.exception.model import InvalidUriException, UnknownHashTypeException
 from cyclonedx.model import ExternalReference, ExternalReferenceType, HashType, Property, XsUri
-from cyclonedx.model.component import Component, ComponentType
+from cyclonedx.model.component import Component, ComponentScope, ComponentType
 from packageurl import PackageURL
 
 from . import BomBuilder, PropertyName, PurlTypePypi
@@ -160,19 +160,30 @@ def _make_bom(self, root_c: Optional['Component'],
         if root_c:
             # root for possible self-installs
             all_components[normalize_packagename(root_c.name)] = root_c
+        non_develop_packages = {
+            normalize_packagename(pkg_name)
+            for gname, pkgs in locker.items()
+            if gname != self.__LOCKFILE_META and gname != 'develop' and isinstance(pkgs, dict)
+            for pkg_name in pkgs
+        }
         for group_name in use_groups:
             self._logger.debug('processing group %r ...', group_name)
             for package_name, package_data in locker.get(group_name, {}).items():
                 package_name_normalized = normalize_packagename(package_name)
                 if package_name_normalized in all_components:
                     component = all_components[package_name_normalized]
                     self._logger.info('existing component for package %r', package_name)
+                    if group_name != 'develop':
+                        component.scope = None
                 else:
                     component = all_components[package_name_normalized] = Component(
                         bom_ref=f'{package_name}{package_data.get("version", "")}',
                         type=ComponentType.LIBRARY,
                         name=package_name,
                         version=package_data['version'][2:] if 'version' in package_data else None,
+                        scope=ComponentScope.EXCLUDED if (
+                            group_name == 'develop' and package_name_normalized not in non_develop_packages
+                        ) else None,
                         external_references=self.__make_extrefs(package_name, package_data, source_urls),
                     )
                     component.purl = PackageURL(

cyclonedx_py/_internal/poetry.py

@@ -292,7 +292,9 @@ def _make_bom(self, project: 'T_NameDict', locker: 'T_NameDict',
 
         use_extras_dep_names = frozenset(map(normalize_packagename,
                                              chain.from_iterable(po_cfg['extras'][e] for e in use_extras)))
-        for group_name in use_groups:
+        sorted_groups = sorted(use_groups, key=lambda g: 1 if g == 'dev' else 0)
+        for group_name in sorted_groups:
+            is_dev = group_name == 'dev'
             for dep_name, dep_specs in po_cfg['group'][group_name].get('dependencies', {}).items():
                 dep_name = normalize_packagename(dep_name)
                 if not isinstance(dep_specs, list):
@@ -319,16 +321,20 @@ def _make_bom(self, project: 'T_NameDict', locker: 'T_NameDict',
                     self.__add_dep(
                         bom, lock_entry,
                         chain.from_iterable(ds.get('extras', ()) for ds in dep_specs),
-                        lock_data)
+                        lock_data,
+                        is_dev=is_dev)
 
         return bom
 
-    def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str], lock_data: 'T_LockData') -> None:
+    def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str], lock_data: 'T_LockData',
+                  *, is_dev: bool = False) -> None:
         if lock_entry.added2bom:
             self._logger.debug('existing component: %r', lock_entry.component)
             lock_entry_dep = None
         else:
             lock_entry.added2bom = True
+            if is_dev:
+                lock_entry.component.scope = ComponentScope.EXCLUDED
             self._logger.info('add component for package %r', lock_entry.name)
             self._logger.debug('add component: %r', lock_entry.component)
             bom.components.add(lock_entry.component)
@@ -347,7 +353,7 @@ def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str
                     continue
                 for dep_lock_entry in dep_lock_entries:
                     lock_entry_dep.dependencies.add(Dependency(dep_lock_entry.component.bom_ref))
-                    self.__add_dep(bom, dep_lock_entry, dep_spec.get('extras', ()), lock_data)
+                    self.__add_dep(bom, dep_lock_entry, dep_spec.get('extras', ()), lock_data, is_dev=is_dev)
         if use_extras:
             use_extras = frozenset(map(normalize_packagename, use_extras))
             lock_entry.component.properties.update(
@@ -369,7 +375,7 @@ def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str
                     continue
                 for dep_lock_entry in dep_lock_entries:
                     lock_entry_dep.dependencies.add(Dependency(dep_lock_entry.component.bom_ref))
-                    self.__add_dep(bom, dep_lock_entry, req.extras, lock_data)
+                    self.__add_dep(bom, dep_lock_entry, req.extras, lock_data, is_dev=is_dev)
 
     @staticmethod
     def _get_lockfile_version(locker: 'T_NameDict') -> tuple[int, ...]:

tests/unit/test_pipenv.py

@@ -0,0 +1,120 @@
+# This file is part of CycloneDX Python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+import logging
+from unittest import TestCase
+
+from cyclonedx.model.component import Component, ComponentScope, ComponentType
+
+from cyclonedx_py._internal.pipenv import PipenvBB
+
+
+class TestPipenvScope(TestCase):
+
+    def setUp(self) -> None:
+        self.bb = PipenvBB(logger=logging.getLogger('test-pipenv'), pypi_url=None)
+
+    def test_dev_packages_marked_excluded(self) -> None:
+        locker = {
+            '_meta': {},
+            'default': {'colorama': {'version': '==0.4.6'}},
+            'develop': {'arrow': {'version': '==1.3.0'}}
+        }
+        bom = self.bb._make_bom(None, locker, frozenset({'default', 'develop'}))
+        comp_map = {c.name: c for c in bom.components}
+
+        self.assertIn('colorama', comp_map)
+        self.assertIsNone(comp_map['colorama'].scope)
+
+        self.assertIn('arrow', comp_map)
+        self.assertEqual(ComponentScope.EXCLUDED, comp_map['arrow'].scope)
+
+    def test_package_in_both_default_and_develop_not_excluded(self) -> None:
+        locker = {
+            '_meta': {},
+            'default': {'shared-pkg': {'version': '==1.0.0'}},
+            'develop': {
+                'shared-pkg': {'version': '==1.0.0'},
+                'dev-only': {'version': '==2.0.0'}
+            }
+        }
+        bom = self.bb._make_bom(None, locker, frozenset({'default', 'develop'}))
+        comp_map = {c.name: c for c in bom.components}
+
+        self.assertIn('shared-pkg', comp_map)
+        self.assertIsNone(comp_map['shared-pkg'].scope)
+
+        self.assertIn('dev-only', comp_map)
+        self.assertEqual(ComponentScope.EXCLUDED, comp_map['dev-only'].scope)
+
+    def test_package_normalized_name_matching(self) -> None:
+        locker = {
+            '_meta': {},
+            'default': {'Types_Python_Dateutil': {'version': '==1.0.0'}},
+            'develop': {'types-python-dateutil': {'version': '==1.0.0'}}
+        }
+        bom = self.bb._make_bom(None, locker, frozenset({'default', 'develop'}))
+        comp_map = {c.name: c for c in bom.components}
+
+        self.assertEqual(1, len(bom.components))
+        comp = next(iter(bom.components))
+        self.assertIsNone(comp.scope)
+
+    def test_default_only_groups_leaves_scope_none(self) -> None:
+        locker = {
+            '_meta': {},
+            'default': {'colorama': {'version': '==0.4.6'}},
+            'develop': {'arrow': {'version': '==1.3.0'}}
+        }
+        bom = self.bb._make_bom(None, locker, frozenset({'default'}))
+        comp_map = {c.name: c for c in bom.components}
+
+        self.assertIn('colorama', comp_map)
+        self.assertIsNone(comp_map['colorama'].scope)
+        self.assertNotIn('arrow', comp_map)
+
+    def test_root_component_scope_not_modified(self) -> None:
+        root_c = Component(name='my-root-pkg', type=ComponentType.APPLICATION)
+        locker = {
+            '_meta': {},
+            'default': {},
+            'develop': {'my-root-pkg': {'version': '==0.1.0'}, 'dev-pkg': {'version': '==1.0.0'}}
+        }
+        bom = self.bb._make_bom(root_c, locker, frozenset({'default', 'develop'}))
+        self.assertIsNone(root_c.scope)
+        self.assertIs(root_c, bom.metadata.component)
+        comp_map = {c.name: c for c in bom.components}
+        self.assertIn('dev-pkg', comp_map)
+        self.assertEqual(ComponentScope.EXCLUDED, comp_map['dev-pkg'].scope)
+
+    def test_custom_non_develop_category_not_excluded(self) -> None:
+        locker = {
+            '_meta': {},
+            'docs': {'sphinx': {'version': '==5.0.0'}},
+            'develop': {
+                'sphinx': {'version': '==5.0.0'},
+                'pytest': {'version': '==7.0.0'}
+            }
+        }
+        bom = self.bb._make_bom(None, locker, frozenset({'docs', 'develop'}))
+        comp_map = {c.name: c for c in bom.components}
+
+        self.assertIn('sphinx', comp_map)
+        self.assertIsNone(comp_map['sphinx'].scope)
+
+        self.assertIn('pytest', comp_map)
+        self.assertEqual(ComponentScope.EXCLUDED, comp_map['pytest'].scope)

tests/unit/test_poetry.py

@@ -0,0 +1,170 @@
+# This file is part of CycloneDX Python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+import logging
+from unittest import TestCase
+
+from cyclonedx.model.component import ComponentScope, ComponentType
+
+from cyclonedx_py._internal.poetry import PoetryBB
+
+
+class TestPoetryComponentScope(TestCase):
+
+    def setUp(self) -> None:
+        self.bb = PoetryBB(logger=logging.getLogger('test-poetry'))
+
+    def test_dev_dependencies_marked_excluded(self) -> None:
+        project = {
+            'tool': {
+                'poetry': {
+                    'name': 'demo',
+                    'version': '1.0.0',
+                    'group': {
+                        'main': {'dependencies': {'prod-pkg': '1.0'}},
+                        'dev': {'dependencies': {'dev-pkg': '1.0'}},
+                    },
+                    'extras': {}
+                }
+            }
+        }
+        locker = {
+            'metadata': {'lock-version': '2.0'},
+            'package': [
+                {'name': 'prod-pkg', 'version': '1.0', 'dependencies': {}},
+                {'name': 'dev-pkg', 'version': '1.0', 'dependencies': {}},
+            ]
+        }
+        bom = self.bb._make_bom(
+            project=project,
+            locker=locker,
+            use_groups=frozenset({'main', 'dev'}),
+            use_extras=frozenset(),
+            mc_type=ComponentType.APPLICATION
+        )
+        c_prod = next(c for c in bom.components if c.name == 'prod-pkg')
+        c_dev = next(c for c in bom.components if c.name == 'dev-pkg')
+
+        self.assertIsNone(c_prod.scope)
+        self.assertEqual(ComponentScope.EXCLUDED, c_dev.scope)
+
+    def test_transitive_dev_dependencies_marked_excluded(self) -> None:
+        project = {
+            'tool': {
+                'poetry': {
+                    'name': 'demo',
+                    'version': '1.0.0',
+                    'group': {
+                        'dev': {'dependencies': {'pytest': '7.0'}},
+                    },
+                    'extras': {}
+                }
+            }
+        }
+        locker = {
+            'metadata': {'lock-version': '2.0'},
+            'package': [
+                {'name': 'pytest', 'version': '7.0', 'dependencies': {'iniconfig': '1.0'}},
+                {'name': 'iniconfig', 'version': '1.0', 'dependencies': {}},
+            ]
+        }
+        bom = self.bb._make_bom(
+            project=project,
+            locker=locker,
+            use_groups=frozenset({'dev'}),
+            use_extras=frozenset(),
+            mc_type=ComponentType.APPLICATION
+        )
+        c_pytest = next(c for c in bom.components if c.name == 'pytest')
+        c_iniconfig = next(c for c in bom.components if c.name == 'iniconfig')
+
+        self.assertEqual(ComponentScope.EXCLUDED, c_pytest.scope)
+        self.assertEqual(ComponentScope.EXCLUDED, c_iniconfig.scope)
+
+    def test_shared_dependency_between_main_and_dev_remains_required(self) -> None:
+        project = {
+            'tool': {
+                'poetry': {
+                    'name': 'demo',
+                    'version': '1.0.0',
+                    'group': {
+                        'main': {'dependencies': {'api-server': '1.0'}},
+                        'dev': {'dependencies': {'test-tool': '1.0'}},
+                    },
+                    'extras': {}
+                }
+            }
+        }
+        locker = {
+            'metadata': {'lock-version': '2.0'},
+            'package': [
+                {'name': 'api-server', 'version': '1.0', 'dependencies': {'pydantic': '2.0'}},
+                {'name': 'test-tool', 'version': '1.0', 'dependencies': {'pydantic': '2.0', 'flake8': '5.0'}},
+                {'name': 'pydantic', 'version': '2.0', 'dependencies': {}},
+                {'name': 'flake8', 'version': '5.0', 'dependencies': {}},
+            ]
+        }
+        bom = self.bb._make_bom(
+            project=project,
+            locker=locker,
+            use_groups=frozenset({'main', 'dev'}),
+            use_extras=frozenset(),
+            mc_type=ComponentType.APPLICATION
+        )
+        c_api = next(c for c in bom.components if c.name == 'api-server')
+        c_pydantic = next(c for c in bom.components if c.name == 'pydantic')
+        c_test = next(c for c in bom.components if c.name == 'test-tool')
+        c_flake = next(c for c in bom.components if c.name == 'flake8')
+
+        self.assertIsNone(c_api.scope)
+        self.assertIsNone(c_pydantic.scope)  # Shared dependency remains runtime
+        self.assertEqual(ComponentScope.EXCLUDED, c_test.scope)
+        self.assertEqual(ComponentScope.EXCLUDED, c_flake.scope)
+
+    def test_legacy_dev_dependencies_category_mapping(self) -> None:
+        project = {
+            'tool': {
+                'poetry': {
+                    'name': 'demo',
+                    'version': '1.0.0',
+                    'group': {
+                        'main': {'dependencies': {'prod-lib': '1.0'}},
+                        'dev': {'dependencies': {'dev-lib': '1.0'}},
+                    },
+                    'extras': {}
+                }
+            }
+        }
+        locker = {
+            'metadata': {'lock-version': '1.1'},
+            'package': [
+                {'name': 'prod-lib', 'version': '1.0', 'category': 'main', 'dependencies': {}},
+                {'name': 'dev-lib', 'version': '1.0', 'category': 'dev', 'dependencies': {}},
+            ]
+        }
+        bom = self.bb._make_bom(
+            project=project,
+            locker=locker,
+            use_groups=frozenset({'main', 'dev'}),
+            use_extras=frozenset(),
+            mc_type=ComponentType.APPLICATION
+        )
+        c_prod = next(c for c in bom.components if c.name == 'prod-lib')
+        c_dev = next(c for c in bom.components if c.name == 'dev-lib')
+
+        self.assertIsNone(c_prod.scope)
+        self.assertEqual(ComponentScope.EXCLUDED, c_dev.scope)