From 3d3d712d3ffb8e80498e34f5095253629914ae49 Mon Sep 17 00:00:00 2001 From: dedsec-terminal Date: Wed, 14 Oct 2026 09:43:19 +0530 Subject: [PATCH] feat(parser): mark dev-dependencies component.scope as "excluded" (#675) Signed-off-by: dedsec-terminal --- cyclonedx_py/_internal/pipenv.py | 13 ++- cyclonedx_py/_internal/poetry.py | 16 ++- tests/unit/test_pipenv.py | 120 ++++++++++++++++++++++ tests/unit/test_poetry.py | 170 +++++++++++++++++++++++++++++++ 4 files changed, 313 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_pipenv.py create mode 100644 tests/unit/test_poetry.py diff --git a/cyclonedx_py/_internal/pipenv.py b/cyclonedx_py/_internal/pipenv.py index 0c1d5997d..dc2a7d002 100644 --- a/cyclonedx_py/_internal/pipenv.py +++ b/cyclonedx_py/_internal/pipenv.py @@ -26,7 +26,7 @@ from cyclonedx.exception.model import InvalidUriException, UnknownHashTypeException from cyclonedx.model import ExternalReference, ExternalReferenceType, HashType, Property, XsUri -from cyclonedx.model.component import Component, ComponentType +from cyclonedx.model.component import Component, ComponentScope, ComponentType from packageurl import PackageURL from . import BomBuilder, PropertyName, PurlTypePypi @@ -160,6 +160,12 @@ def _make_bom(self, root_c: Optional['Component'], if root_c: # root for possible self-installs all_components[normalize_packagename(root_c.name)] = root_c + non_develop_packages = { + normalize_packagename(pkg_name) + for gname, pkgs in locker.items() + if gname != self.__LOCKFILE_META and gname != 'develop' and isinstance(pkgs, dict) + for pkg_name in pkgs + } for group_name in use_groups: self._logger.debug('processing group %r ...', group_name) for package_name, package_data in locker.get(group_name, {}).items(): @@ -167,12 +173,17 @@ def _make_bom(self, root_c: Optional['Component'], if package_name_normalized in all_components: component = all_components[package_name_normalized] self._logger.info('existing component for package %r', package_name) + if group_name != 'develop': + component.scope = None else: component = all_components[package_name_normalized] = Component( bom_ref=f'{package_name}{package_data.get("version", "")}', type=ComponentType.LIBRARY, name=package_name, version=package_data['version'][2:] if 'version' in package_data else None, + scope=ComponentScope.EXCLUDED if ( + group_name == 'develop' and package_name_normalized not in non_develop_packages + ) else None, external_references=self.__make_extrefs(package_name, package_data, source_urls), ) component.purl = PackageURL( diff --git a/cyclonedx_py/_internal/poetry.py b/cyclonedx_py/_internal/poetry.py index d3eaa4937..aec38e11c 100644 --- a/cyclonedx_py/_internal/poetry.py +++ b/cyclonedx_py/_internal/poetry.py @@ -292,7 +292,9 @@ def _make_bom(self, project: 'T_NameDict', locker: 'T_NameDict', use_extras_dep_names = frozenset(map(normalize_packagename, chain.from_iterable(po_cfg['extras'][e] for e in use_extras))) - for group_name in use_groups: + sorted_groups = sorted(use_groups, key=lambda g: 1 if g == 'dev' else 0) + for group_name in sorted_groups: + is_dev = group_name == 'dev' for dep_name, dep_specs in po_cfg['group'][group_name].get('dependencies', {}).items(): dep_name = normalize_packagename(dep_name) if not isinstance(dep_specs, list): @@ -319,16 +321,20 @@ def _make_bom(self, project: 'T_NameDict', locker: 'T_NameDict', self.__add_dep( bom, lock_entry, chain.from_iterable(ds.get('extras', ()) for ds in dep_specs), - lock_data) + lock_data, + is_dev=is_dev) return bom - def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str], lock_data: 'T_LockData') -> None: + def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str], lock_data: 'T_LockData', + *, is_dev: bool = False) -> None: if lock_entry.added2bom: self._logger.debug('existing component: %r', lock_entry.component) lock_entry_dep = None else: lock_entry.added2bom = True + if is_dev: + lock_entry.component.scope = ComponentScope.EXCLUDED self._logger.info('add component for package %r', lock_entry.name) self._logger.debug('add component: %r', lock_entry.component) bom.components.add(lock_entry.component) @@ -347,7 +353,7 @@ def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str continue for dep_lock_entry in dep_lock_entries: lock_entry_dep.dependencies.add(Dependency(dep_lock_entry.component.bom_ref)) - self.__add_dep(bom, dep_lock_entry, dep_spec.get('extras', ()), lock_data) + self.__add_dep(bom, dep_lock_entry, dep_spec.get('extras', ()), lock_data, is_dev=is_dev) if use_extras: use_extras = frozenset(map(normalize_packagename, use_extras)) lock_entry.component.properties.update( @@ -369,7 +375,7 @@ def __add_dep(self, bom: 'Bom', lock_entry: _LockEntry, use_extras: Iterable[str continue for dep_lock_entry in dep_lock_entries: lock_entry_dep.dependencies.add(Dependency(dep_lock_entry.component.bom_ref)) - self.__add_dep(bom, dep_lock_entry, req.extras, lock_data) + self.__add_dep(bom, dep_lock_entry, req.extras, lock_data, is_dev=is_dev) @staticmethod def _get_lockfile_version(locker: 'T_NameDict') -> tuple[int, ...]: diff --git a/tests/unit/test_pipenv.py b/tests/unit/test_pipenv.py new file mode 100644 index 000000000..1546e5e32 --- /dev/null +++ b/tests/unit/test_pipenv.py @@ -0,0 +1,120 @@ +# This file is part of CycloneDX Python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) OWASP Foundation. All Rights Reserved. + +import logging +from unittest import TestCase + +from cyclonedx.model.component import Component, ComponentScope, ComponentType + +from cyclonedx_py._internal.pipenv import PipenvBB + + +class TestPipenvScope(TestCase): + + def setUp(self) -> None: + self.bb = PipenvBB(logger=logging.getLogger('test-pipenv'), pypi_url=None) + + def test_dev_packages_marked_excluded(self) -> None: + locker = { + '_meta': {}, + 'default': {'colorama': {'version': '==0.4.6'}}, + 'develop': {'arrow': {'version': '==1.3.0'}} + } + bom = self.bb._make_bom(None, locker, frozenset({'default', 'develop'})) + comp_map = {c.name: c for c in bom.components} + + self.assertIn('colorama', comp_map) + self.assertIsNone(comp_map['colorama'].scope) + + self.assertIn('arrow', comp_map) + self.assertEqual(ComponentScope.EXCLUDED, comp_map['arrow'].scope) + + def test_package_in_both_default_and_develop_not_excluded(self) -> None: + locker = { + '_meta': {}, + 'default': {'shared-pkg': {'version': '==1.0.0'}}, + 'develop': { + 'shared-pkg': {'version': '==1.0.0'}, + 'dev-only': {'version': '==2.0.0'} + } + } + bom = self.bb._make_bom(None, locker, frozenset({'default', 'develop'})) + comp_map = {c.name: c for c in bom.components} + + self.assertIn('shared-pkg', comp_map) + self.assertIsNone(comp_map['shared-pkg'].scope) + + self.assertIn('dev-only', comp_map) + self.assertEqual(ComponentScope.EXCLUDED, comp_map['dev-only'].scope) + + def test_package_normalized_name_matching(self) -> None: + locker = { + '_meta': {}, + 'default': {'Types_Python_Dateutil': {'version': '==1.0.0'}}, + 'develop': {'types-python-dateutil': {'version': '==1.0.0'}} + } + bom = self.bb._make_bom(None, locker, frozenset({'default', 'develop'})) + comp_map = {c.name: c for c in bom.components} + + self.assertEqual(1, len(bom.components)) + comp = next(iter(bom.components)) + self.assertIsNone(comp.scope) + + def test_default_only_groups_leaves_scope_none(self) -> None: + locker = { + '_meta': {}, + 'default': {'colorama': {'version': '==0.4.6'}}, + 'develop': {'arrow': {'version': '==1.3.0'}} + } + bom = self.bb._make_bom(None, locker, frozenset({'default'})) + comp_map = {c.name: c for c in bom.components} + + self.assertIn('colorama', comp_map) + self.assertIsNone(comp_map['colorama'].scope) + self.assertNotIn('arrow', comp_map) + + def test_root_component_scope_not_modified(self) -> None: + root_c = Component(name='my-root-pkg', type=ComponentType.APPLICATION) + locker = { + '_meta': {}, + 'default': {}, + 'develop': {'my-root-pkg': {'version': '==0.1.0'}, 'dev-pkg': {'version': '==1.0.0'}} + } + bom = self.bb._make_bom(root_c, locker, frozenset({'default', 'develop'})) + self.assertIsNone(root_c.scope) + self.assertIs(root_c, bom.metadata.component) + comp_map = {c.name: c for c in bom.components} + self.assertIn('dev-pkg', comp_map) + self.assertEqual(ComponentScope.EXCLUDED, comp_map['dev-pkg'].scope) + + def test_custom_non_develop_category_not_excluded(self) -> None: + locker = { + '_meta': {}, + 'docs': {'sphinx': {'version': '==5.0.0'}}, + 'develop': { + 'sphinx': {'version': '==5.0.0'}, + 'pytest': {'version': '==7.0.0'} + } + } + bom = self.bb._make_bom(None, locker, frozenset({'docs', 'develop'})) + comp_map = {c.name: c for c in bom.components} + + self.assertIn('sphinx', comp_map) + self.assertIsNone(comp_map['sphinx'].scope) + + self.assertIn('pytest', comp_map) + self.assertEqual(ComponentScope.EXCLUDED, comp_map['pytest'].scope) diff --git a/tests/unit/test_poetry.py b/tests/unit/test_poetry.py new file mode 100644 index 000000000..01887303c --- /dev/null +++ b/tests/unit/test_poetry.py @@ -0,0 +1,170 @@ +# This file is part of CycloneDX Python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) OWASP Foundation. All Rights Reserved. + +import logging +from unittest import TestCase + +from cyclonedx.model.component import ComponentScope, ComponentType + +from cyclonedx_py._internal.poetry import PoetryBB + + +class TestPoetryComponentScope(TestCase): + + def setUp(self) -> None: + self.bb = PoetryBB(logger=logging.getLogger('test-poetry')) + + def test_dev_dependencies_marked_excluded(self) -> None: + project = { + 'tool': { + 'poetry': { + 'name': 'demo', + 'version': '1.0.0', + 'group': { + 'main': {'dependencies': {'prod-pkg': '1.0'}}, + 'dev': {'dependencies': {'dev-pkg': '1.0'}}, + }, + 'extras': {} + } + } + } + locker = { + 'metadata': {'lock-version': '2.0'}, + 'package': [ + {'name': 'prod-pkg', 'version': '1.0', 'dependencies': {}}, + {'name': 'dev-pkg', 'version': '1.0', 'dependencies': {}}, + ] + } + bom = self.bb._make_bom( + project=project, + locker=locker, + use_groups=frozenset({'main', 'dev'}), + use_extras=frozenset(), + mc_type=ComponentType.APPLICATION + ) + c_prod = next(c for c in bom.components if c.name == 'prod-pkg') + c_dev = next(c for c in bom.components if c.name == 'dev-pkg') + + self.assertIsNone(c_prod.scope) + self.assertEqual(ComponentScope.EXCLUDED, c_dev.scope) + + def test_transitive_dev_dependencies_marked_excluded(self) -> None: + project = { + 'tool': { + 'poetry': { + 'name': 'demo', + 'version': '1.0.0', + 'group': { + 'dev': {'dependencies': {'pytest': '7.0'}}, + }, + 'extras': {} + } + } + } + locker = { + 'metadata': {'lock-version': '2.0'}, + 'package': [ + {'name': 'pytest', 'version': '7.0', 'dependencies': {'iniconfig': '1.0'}}, + {'name': 'iniconfig', 'version': '1.0', 'dependencies': {}}, + ] + } + bom = self.bb._make_bom( + project=project, + locker=locker, + use_groups=frozenset({'dev'}), + use_extras=frozenset(), + mc_type=ComponentType.APPLICATION + ) + c_pytest = next(c for c in bom.components if c.name == 'pytest') + c_iniconfig = next(c for c in bom.components if c.name == 'iniconfig') + + self.assertEqual(ComponentScope.EXCLUDED, c_pytest.scope) + self.assertEqual(ComponentScope.EXCLUDED, c_iniconfig.scope) + + def test_shared_dependency_between_main_and_dev_remains_required(self) -> None: + project = { + 'tool': { + 'poetry': { + 'name': 'demo', + 'version': '1.0.0', + 'group': { + 'main': {'dependencies': {'api-server': '1.0'}}, + 'dev': {'dependencies': {'test-tool': '1.0'}}, + }, + 'extras': {} + } + } + } + locker = { + 'metadata': {'lock-version': '2.0'}, + 'package': [ + {'name': 'api-server', 'version': '1.0', 'dependencies': {'pydantic': '2.0'}}, + {'name': 'test-tool', 'version': '1.0', 'dependencies': {'pydantic': '2.0', 'flake8': '5.0'}}, + {'name': 'pydantic', 'version': '2.0', 'dependencies': {}}, + {'name': 'flake8', 'version': '5.0', 'dependencies': {}}, + ] + } + bom = self.bb._make_bom( + project=project, + locker=locker, + use_groups=frozenset({'main', 'dev'}), + use_extras=frozenset(), + mc_type=ComponentType.APPLICATION + ) + c_api = next(c for c in bom.components if c.name == 'api-server') + c_pydantic = next(c for c in bom.components if c.name == 'pydantic') + c_test = next(c for c in bom.components if c.name == 'test-tool') + c_flake = next(c for c in bom.components if c.name == 'flake8') + + self.assertIsNone(c_api.scope) + self.assertIsNone(c_pydantic.scope) # Shared dependency remains runtime + self.assertEqual(ComponentScope.EXCLUDED, c_test.scope) + self.assertEqual(ComponentScope.EXCLUDED, c_flake.scope) + + def test_legacy_dev_dependencies_category_mapping(self) -> None: + project = { + 'tool': { + 'poetry': { + 'name': 'demo', + 'version': '1.0.0', + 'group': { + 'main': {'dependencies': {'prod-lib': '1.0'}}, + 'dev': {'dependencies': {'dev-lib': '1.0'}}, + }, + 'extras': {} + } + } + } + locker = { + 'metadata': {'lock-version': '1.1'}, + 'package': [ + {'name': 'prod-lib', 'version': '1.0', 'category': 'main', 'dependencies': {}}, + {'name': 'dev-lib', 'version': '1.0', 'category': 'dev', 'dependencies': {}}, + ] + } + bom = self.bb._make_bom( + project=project, + locker=locker, + use_groups=frozenset({'main', 'dev'}), + use_extras=frozenset(), + mc_type=ComponentType.APPLICATION + ) + c_prod = next(c for c in bom.components if c.name == 'prod-lib') + c_dev = next(c for c in bom.components if c.name == 'dev-lib') + + self.assertIsNone(c_prod.scope) + self.assertEqual(ComponentScope.EXCLUDED, c_dev.scope)