diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 68094c88a4..9fa5d3c7c5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Unreleased * feat: expose the ``is_addon`` attribute in the pipelines REST API * feat: [code-genetics] determine the origin of files from combined origin matches #1929 by @tdruez +* feat: [code-genetics] resolve conflicting origins to detect the ultimate origin of files #1930 by @tdruez v38.2.0 (2026-10-22) -------------------- diff --git a/docs/application-settings.rst b/docs/application-settings.rst index 617b52e902..4442896905 100644 --- a/docs/application-settings.rst +++ b/docs/application-settings.rst @@ -381,6 +381,13 @@ API key using ``MATCHCODEIO_API_KEY``:: MATCHCODEIO_API_KEY=insert_your_api_key_here +.. note:: + MatchCode.io does not report the path of a matched file in its package. + The :ref:`pipeline_find_origins` pipeline fetches this path from PurlDB when + :ref:`scancodeio_settings_purldb` is configured. Without it, the + ``containment`` rule of the :ref:`origin_determination` cannot apply to the + MatchCode.io matches. + .. _scancodeio_settings_scanoss: SCANOSS diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 62f54af556..e5eb8bbf4b 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -150,6 +150,10 @@ Find Origins (addon) :ref:`scancodeio_settings_scanoss` to configure access to MatchCode.io and SCANOSS in your ScanCode.io instance. +.. seealso:: + :ref:`origin_determination` for the rules that determine the origin of each + file and directory. + .. autoclass:: scanpipe.pipelines.find_origins.FindOrigins() :members: :member-order: bysource diff --git a/docs/data-models.rst b/docs/data-models.rst index b6edbff143..31a0ff4e08 100644 --- a/docs/data-models.rst +++ b/docs/data-models.rst @@ -63,6 +63,9 @@ OriginMatch OriginDetermination ------------------- +.. seealso:: + :ref:`origin_determination` for the statuses and the values of ``rule``. + .. autoclass:: scanpipe.models.OriginDetermination() :members: :undoc-members: diff --git a/docs/index.rst b/docs/index.rst index 234542e246..ac27ec3023 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -124,6 +124,7 @@ Indices and tables tutorial_web_ui_scan_repo_health scanpipe-concepts built-in-pipelines + origin-determination custom-pipelines scanpipe-pipes project-configuration diff --git a/docs/origin-determination.rst b/docs/origin-determination.rst new file mode 100644 index 0000000000..f9037fdaa4 --- /dev/null +++ b/docs/origin-determination.rst @@ -0,0 +1,167 @@ +.. _origin_determination: + +Origin Determination +==================== + +The :ref:`pipeline_find_origins` pipeline stores the candidate origin packages +reported by each origin detection tool as origin matches, see +:ref:`data_models_origin_match`. Its last step combines these matches into one +origin determination per matched file or directory, see +:ref:`data_models_origin_determination`. + +Each determination has a status and the name of the rule that set it. Only an +agreement between two tools confirms an origin automatically. Any other case +needs a review, with a proposed origin when a rule can select one. + +Candidates +---------- + +The origin matches of a file or directory are grouped by package into +candidates: + +- The matches of the same package version are one candidate, even when the tools + report different PURLs, such as ``pkg:github/benjaminp/six@1.17.0`` and + ``pkg:pypi/six@1.17.0``, as SCANOSS reports the equivalent PURLs of a + component. A version with a ``v`` prefix, such as the ``v1.0.3`` Git tag, is + the same as the ``1.0.3`` release. +- The snippet matches covering fewer than 20 lines of the file are not stored + as origin matches, such as a common license header found in many unrelated + packages. The raw tool results are kept in the project output files. +- When nested directories are matched to the same package, the deepest + directory is kept as the most precise match. +- A file matched only to the package of its closest matched directory has no + determination of its own: its matches count for the directory. + +Statuses +-------- + +``auto-confirmed`` + Set by the ``confirmed-by-two-tools`` rule. + +``needs-review`` + Set by any other rule. The PURL is the proposed origin, or empty when no + candidate is preferred. + +``confirmed`` + Set by a user who confirms the origin. + +``no-origin`` + Set by a user when the file or directory has no open source origin. + +Rules +----- + +The rules are evaluated in this order, and the first rule that applies sets the +determination. The candidates that carry the copy of another candidate, as +identified by the ``containment`` rule, are not compared by the next rules. + +.. list-table:: + :header-rows: 1 + :widths: 20 50 15 15 + + * - Rule + - Applies when + - Status + - Proposed origin + * - ``containment`` + - Different packages: the copy in all the candidates but one is in a file, + or in a directory just under a vendor directory, named after another + candidate, such as ``pip/_vendor/six.py`` in pip. The vendor directories + are ``_vendor``, ``vendor``, ``vendored``, ``packages``, + ``third_party``, and ``extern``. + - ``needs-review`` + - The remaining candidate. + * - ``preferred-version`` + - Versions of the same package: one is more similar than all the others by + at least 20 points, or else only one is a release, as opposed to a commit + hash or a missing version, or else one release is more similar than the + other releases by at least 20 points, or else one release was released + before the other releases, as a file often stays the same in several + releases. + - ``needs-review`` + - The preferred version. + * - ``conflicting-versions`` + - Versions of the same package, without a preferred version. + - ``needs-review`` + - None. + * - ``highest-similarity`` + - Different packages: one candidate is more similar than all the others by + at least 20 points, and all the similarities are known. + - ``needs-review`` + - The most similar candidate. + * - ``conflicting-candidates`` + - Different packages, not resolved by the previous rules. + - ``needs-review`` + - None. + * - ``no-exact-match`` + - One candidate, with partial matches only: snippet, approximate, or + directory matches. + - ``needs-review`` + - The candidate. + * - ``single-tool`` + - One candidate, with an exact file or package match, reported by a single + tool. + - ``needs-review`` + - The candidate. + * - ``confirmed-by-two-tools`` + - One candidate, with at least one exact file or package match, reported by + two tools. + - ``auto-confirmed`` + - The candidate. + +Similarity and release date +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- The similarity is the percentage of the file lines matched: the SCANOSS + ``matched`` value, 100 for a MatchCode.io exact match, and the line coverage of + the MatchCode.io snippet matches. It is unknown for the directory and + approximate matches. The tools measure the same copy a few points apart, hence + the 20 points margin. +- The release date is reported by SCANOSS, and by MatchCode.io when PurlDB + knows it. It only selects among the versions of the same package, never + between different packages: the earliest candidate is often another copy, + such as a GitHub repository copying the file, when the original package is + not among the candidates. + +Re-running the pipeline +----------------------- + +- The automatic determinations, ``auto-confirmed`` and ``needs-review``, are + replaced. +- The user determinations, ``confirmed`` and ``no-origin``, are kept. A + directory determined by a user covers its content: the origin matches under it + are not used. + +Origin packages +--------------- + +The files and directories of the ``auto-confirmed`` and ``confirmed`` +determinations are assigned to the package of their PURL. The files of a +determined directory are assigned with it, unless they have a determination of +their own. + +- A package detected by the codebase scan is reused, also when its PURL has + other qualifiers, such as a Maven ``classifier``. +- Otherwise, the package is created from the tools data: download URL, homepage + URL, release date, and declared license. It is identified with the + ``origin_determination`` datasource ID, and created again on each run. + +Limitations +----------- + +- A single tool never confirms an origin automatically, as it can miss other + candidates: SCANOSS reports one component per file, and MatchCode.io only + knows the indexed packages. +- A copy taken from a package that vendors it, such as ``six.py`` copied from + urllib3, is attributed to the vendoring package when no tool reports the + original package. +- MatchCode.io rarely reports a release date, so the earliest release rarely + selects among the versions of its candidates. +- The 20 lines threshold is calibrated on the MIT license header. A longer + header, such as the BSD-3-Clause one, still counts as a snippet match. +- A small file copied with changes, matched as a snippet on fewer than 20 + lines, is ignored. +- The ``containment`` rule needs the path of the copy in each candidate. + SCANOSS only reports the file name, and the path of a MatchCode.io match is + fetched from PurlDB, when configured. When a PurlDB request fails, the paths + are partial and the rule does not apply to the files without a path. diff --git a/scanpipe/pipelines/find_origins.py b/scanpipe/pipelines/find_origins.py index 9e57b0fc74..6bc2ba70ee 100644 --- a/scanpipe/pipelines/find_origins.py +++ b/scanpipe/pipelines/find_origins.py @@ -25,6 +25,7 @@ from scanpipe.pipelines import Pipeline from scanpipe.pipes import matchcode from scanpipe.pipes import origin +from scanpipe.pipes import purldb from scanpipe.pipes import scanoss @@ -39,7 +40,9 @@ class FindOrigins(Pipeline): Supported tools: - MatchCode.io: the codebase scan, including file and directory - fingerprints, is sent to the configured MatchCode.io server. + fingerprints, is sent to the configured MatchCode.io server. When PurlDB is + configured, the paths of the matched files in their package are fetched + from PurlDB. - SCANOSS: the codebase files are fingerprinted locally, then their paths, checksums, sizes, and fingerprints are sent to the configured SCANOSS server. The file contents are not sent. @@ -77,6 +80,12 @@ def find_matchcode_origin_matches(self): ) self.log(f"{origin_match_count:,d} MatchCode.io origin matches processed.") + if not purldb.is_configured(): + self.log( + "PurlDB is not configured: the paths of the files matched by " + "MatchCode.io in their package are not available." + ) + def find_scanoss_origin_matches(self): """Scan the codebase with SCANOSS and store the reported origin matches.""" if not scanoss.is_configured(): @@ -106,7 +115,9 @@ def determine_origins(self): Determine the origin of each matched file and directory from the origin matches of all the tools. An origin is confirmed automatically when two tools report the same package, with at least one exact match. Otherwise, - the origin needs a review. The origins confirmed by a user are kept. + the origin needs a review, with a proposed origin when a rule can select one + among the candidates, such as the candidate vendored by another candidate. + The origins confirmed by a user are kept. The files and directories of a confirmed origin are assigned to the origin package, created from the tools data when the codebase scan did not detect it. diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index c863404c8b..57271d51d9 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -619,6 +619,31 @@ def run_command_safely(command_args, env=None): return completed_process.stdout +# The shorter snippet matches are noise, such as a common license header matching +# unrelated packages: the MIT license header is 19 lines long. Counted in lines, +# not in percentage, to keep a copy of a few dozen lines in a large file. +MIN_SNIPPET_LINE_COUNT = 20 + + +def get_matched_line_count(origin_match): + """Return the number of resource lines covered by the ``origin_match``.""" + matched_lines = { + line + for line_range in origin_match.line_ranges + for line in range(line_range["lines"][0], line_range["lines"][1] + 1) + } + return len(matched_lines) + + +def is_short_snippet_match(origin_match): + """Return True when ``origin_match`` is a snippet match too short to count.""" + is_snippet_match = origin_match.match_type == OriginMatch.MatchType.SNIPPET + return ( + is_snippet_match + and get_matched_line_count(origin_match) < MIN_SNIPPET_LINE_COUNT + ) + + def make_origin_match(resource, tool, tool_version, **origin_match_data): """Return an unsaved OriginMatch of the ``resource`` reported by ``tool``.""" return OriginMatch( @@ -642,6 +667,8 @@ def create_origin_matches( The previous origin matches of ``tool`` are replaced, so a candidate that the tool does not report anymore is not kept. The replacement is atomic: on error, the previous origin matches are kept unchanged. + The snippet matches shorter than ``MIN_SNIPPET_LINE_COUNT`` lines are not + created. The raw tool results are kept in the project output files. Return the number of processed origin matches and the list of paths missing from the project codebase. """ @@ -663,6 +690,7 @@ def create_origin_matches( origin_match for path, resource in resources_by_path.items() for origin_match in make_origin_matches(resource, results_by_path[path]) + if not is_short_snippet_match(origin_match) ] OriginMatch.objects.bulk_create(origin_matches, ignore_conflicts=True) origin_match_count += len(origin_matches) diff --git a/scanpipe/pipes/matchcode.py b/scanpipe/pipes/matchcode.py index 7c3ca6ad27..777b8fb577 100644 --- a/scanpipe/pipes/matchcode.py +++ b/scanpipe/pipes/matchcode.py @@ -24,6 +24,7 @@ import logging from collections import defaultdict from functools import partial +from itertools import batched from pathlib import Path from django.conf import settings @@ -44,6 +45,7 @@ from scanpipe.pipes import flag from scanpipe.pipes import make_origin_match from scanpipe.pipes import poll_until_success +from scanpipe.pipes import purldb from scanpipe.pipes.output import to_json from scanpipe.pipes.scancode import _scan_resource from scanpipe.pipes.scancode import scan_resources @@ -524,6 +526,33 @@ def get_origin_match_extra_data(package_data): } +def merge_matched_snippets(matched_snippets): + """ + Return the ``matched_snippets`` merged by package and origin file, as + MatchCode.io can report several snippets of the same origin file. + The merged snippet has all the matched token positions and the highest + similarity. + """ + merged_snippets = {} + for matched_snippet in matched_snippets: + origin_key = (matched_snippet["package"], matched_snippet["resource"]) + merged_snippet = merged_snippets.get(origin_key) + if not merged_snippet: + merged_snippets[origin_key] = dict(matched_snippet) + continue + + match_detections = { + *merged_snippet["match_detections"], + *matched_snippet["match_detections"], + } + merged_snippet["match_detections"] = sorted(match_detections) + merged_snippet["similarity"] = max( + merged_snippet["similarity"], matched_snippet["similarity"] + ) + + return list(merged_snippets.values()) + + def make_snippet_origin_matches( resource, matched_snippets, package_data_by_purl, tool_version ): @@ -536,7 +565,7 @@ def make_snippet_origin_matches( """ line_by_pos = get_resource_line_by_pos(resource) - for matched_snippet in matched_snippets: + for matched_snippet in merge_matched_snippets(matched_snippets): origin_path = matched_snippet["resource"] if is_legal_file_path(origin_path): continue @@ -566,10 +595,53 @@ def make_snippet_origin_matches( ) +def get_path_depth_and_path(purldb_resource): + path = purldb_resource["path"] + return path.count("/"), path + + +def get_origin_paths(sha1_list, candidate_purls, batch_size=1000): + """ + Return the paths of the ``sha1_list`` files in the ``candidate_purls`` + packages, as a mapping of (package PURL, sha1) to path, as MatchCode.io does + not report the path of a matched file. PurlDB reports the copies of a file in + all its packages: only the candidate packages are kept. When a package has + several copies of a file, the shallowest path is kept. Empty when PurlDB is not + configured. + The sha1 are sent to PurlDB by batches of ``batch_size``. + """ + if not sha1_list or not purldb.is_configured(): + return {} + + purldb_resources = sorted( + ( + purldb_resource + for sha1_batch in batched(sha1_list, batch_size) + for purldb_resource in purldb.match_resources(list(sha1_batch)) + if purldb_resource["purl"] in candidate_purls + ), + key=get_path_depth_and_path, + ) + origin_paths = {} + for purldb_resource in purldb_resources: + origin_key = (purldb_resource["purl"], purldb_resource["sha1"]) + origin_paths.setdefault(origin_key, purldb_resource["path"]) + return origin_paths + + def make_package_origin_matches( - resource, match_type, package_uids, package_data_by_uid, tool_version + resource, + match_type, + package_uids, + package_data_by_uid, + tool_version, + origin_paths, ): - """Yield unsaved OriginMatch instances for the ``resource`` matched packages.""" + """ + Yield unsaved OriginMatch instances for the ``resource`` matched packages. + The ``origin_paths`` mapping of (package PURL, sha1) to path provides the path + of a matched file in its package. + """ similarity = 100 if match_type in OriginMatch.EXACT_MATCH_TYPES else None seen_purls = set() @@ -586,13 +658,19 @@ def make_package_origin_matches( tool_version, purl=purl, match_type=match_type, + origin_path=origin_paths.get((purl, resource.sha1), ""), similarity=similarity, extra_data=get_origin_match_extra_data(package_data), ) def make_origin_matches( - resource, resource_results, package_data_by_uid, package_data_by_purl, tool_version + resource, + resource_results, + package_data_by_uid, + package_data_by_purl, + tool_version, + origin_paths, ): """Yield unsaved OriginMatch instances from a ``resource`` match results.""" matched_snippets = (resource_results.get("extra_data") or {}).get( @@ -614,13 +692,34 @@ def make_origin_matches( resource_results["for_packages"], package_data_by_uid, tool_version, + origin_paths, ) +def get_matched_file_sha1s(project, results_by_path): + """ + Return the distinct sha1 of the ``project`` files matched to a PurlDB file. + The legal files are excluded, as a common license file is found in many + packages. + """ + matched_file_paths = [ + path + for path, resource_results in results_by_path.items() + if resource_results.get("status") == flag.MATCHED_TO_PURLDB_RESOURCE + and not is_legal_file_path(path) + ] + matched_files = project.codebaseresources.filter( + path__in=matched_file_paths + ).exclude(sha1="") + matched_file_sha1s = matched_files.order_by("sha1").values_list("sha1", flat=True) + return list(matched_file_sha1s.distinct()) + + def load_matchcode_results(project, match_results): """ Create OriginMatch records for the ``project`` resources from the MatchCode.io - ``match_results``. Return the number of processed origin matches. + ``match_results``. The path of the matched files in their package is fetched + from PurlDB when configured. Return the number of processed origin matches. """ packages_data = match_results.get("packages", []) package_data_by_uid = { @@ -634,11 +733,13 @@ def load_matchcode_results(project, match_results): for resource_results in match_results.get("files", []) if resource_results.get("for_packages") } + matched_file_sha1s = get_matched_file_sha1s(project, results_by_path) make_matchcode_origin_matches = partial( make_origin_matches, package_data_by_uid=package_data_by_uid, package_data_by_purl=package_data_by_purl, tool_version=match_results["headers"][0]["tool_version"], + origin_paths=get_origin_paths(matched_file_sha1s, package_data_by_purl.keys()), ) origin_match_count, _ = create_origin_matches( diff --git a/scanpipe/pipes/origin.py b/scanpipe/pipes/origin.py index 9517a1e62c..6409097364 100644 --- a/scanpipe/pipes/origin.py +++ b/scanpipe/pipes/origin.py @@ -20,10 +20,13 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import re from contextlib import suppress from dataclasses import dataclass +from datetime import date from itertools import batched from itertools import groupby +from itertools import pairwise from operator import attrgetter from pathlib import PurePosixPath @@ -38,8 +41,11 @@ from scanpipe.models import OriginMatch from scanpipe.pipes.resolve import convert_spdx_expression -CONFLICTING_CANDIDATES_RULE = "conflicting-candidates" +CONTAINMENT_RULE = "containment" +PREFERRED_VERSION_RULE = "preferred-version" CONFLICTING_VERSIONS_RULE = "conflicting-versions" +HIGHEST_SIMILARITY_RULE = "highest-similarity" +CONFLICTING_CANDIDATES_RULE = "conflicting-candidates" NO_EXACT_MATCH_RULE = "no-exact-match" SINGLE_TOOL_RULE = "single-tool" CONFIRMED_BY_TWO_TOOLS_RULE = "confirmed-by-two-tools" @@ -58,6 +64,26 @@ # codebase scan. SCANOSS_COMPONENT_LICENSE_SOURCES = ["component_declared", "license_file"] +# The tools measure the similarity of a copy differently, a few points apart: a +# modified copy of inih "ini.c" is 95% similar for MatchCode.io and 94% for SCANOSS. +# A candidate is more similar than another only beyond this margin, in points. +MIN_SIMILARITY_MARGIN = 20 + +# A Git commit hash, full or abbreviated, such as the "498f34b" version reported by +# SCANOSS for a modified copy of inih. +COMMIT_HASH_PATTERN = re.compile(r"[0-9a-f]{7,40}") + +# The directories where a package keeps its copies of other packages, such as +# "pip/_vendor/requests/" or "urllib3/packages/six.py". +VENDOR_DIRECTORY_NAMES = { + "_vendor", + "vendor", + "vendored", + "packages", + "third_party", + "extern", +} + def get_package_purl(purl, version=None): """ @@ -85,14 +111,76 @@ def get_versioned_purls(purls, version): return versioned_purls +def get_equivalent_versions(version): + """ + Return the ``version`` and its form without "v" prefix, such as "1.0.3" for + the "v1.0.3" Git tag reported by SCANOSS for the "1.0.3" npm release. + """ + has_v_prefix = bool(version) and version[0] == "v" and version[1:2].isdigit() + if has_v_prefix: + return [version, version[1:]] + return [version] + + def get_equivalent_purls(origin_match): """ Return the PURLs identifying the ``origin_match`` package: its own PURL and the equivalent PURLs reported by the tool, such as the SCANOSS ``purls``. """ version = PackageURL.from_string(origin_match.purl).version - tool_purls = origin_match.extra_data.get("purls", []) - return get_versioned_purls([origin_match.purl, *tool_purls], version) + purls = [origin_match.purl, *origin_match.extra_data.get("purls", [])] + return { + equivalent_purl + for equivalent_version in get_equivalent_versions(version) + for equivalent_purl in get_versioned_purls(purls, equivalent_version) + } + + +def is_release_version(version): + """ + Return True when the ``version`` is a release, as opposed to a commit hash or + a missing version, such as a SCANOSS component without version. + """ + if not version: + return False + + # A date version, such as "20240101", is not a commit hash. + return version.isdigit() or not COMMIT_HASH_PATTERN.fullmatch(version) + + +def normalize_name(name): + """Return the ``name`` of a package or path segment in a comparable form.""" + return name.lower().replace("-", "_") + + +def get_copy_names(path): + """ + Return the normalized names that can identify the package of the file at + ``path``: its file name without extensions, such as "jquery.min" and "jquery" + for "jquery.min.js", and the directories just under a vendor directory, such + as "requests" for "pip/_vendor/requests/models.py". + """ + file_path = PurePosixPath(path) + names = {file_path.stem, file_path.name.partition(".")[0]} + + for directory_name, child_directory_name in pairwise(file_path.parent.parts): + if normalize_name(directory_name) in VENDOR_DIRECTORY_NAMES: + names.add(child_directory_name) + + return {normalize_name(name) for name in names} + + +def get_release_date(origin_match): + """ + Return the release date of the ``origin_match`` package, None when the tool + does not report a valid date. + """ + release_date = origin_match.extra_data.get("release_date") + if not release_date: + return + + with suppress(ValueError): + return date.fromisoformat(release_date) def get_versionless_purl(purl): @@ -153,6 +241,47 @@ def purl(self): def versionless_purls(self): return {get_versionless_purl(purl) for purl in self.equivalent_purls} + @property + def package_names(self): + return { + normalize_name(PackageURL.from_string(purl).name) + for purl in self.equivalent_purls + } + + @property + def copy_names(self): + """Return the names identifying the copies in the origin package.""" + return { + name + for origin_match in self.origin_matches + if origin_match.origin_path + for name in get_copy_names(origin_match.origin_path) + } + + @property + def is_release_version(self): + return is_release_version(PackageURL.from_string(self.purl).version) + + @property + def similarity(self): + """Return the highest similarity of the origin matches, None if unknown.""" + similarities = [ + origin_match.similarity + for origin_match in self.origin_matches + if origin_match.similarity is not None + ] + return max(similarities, default=None) + + @property + def release_date(self): + """Return the earliest release date of the origin matches, None if unknown.""" + release_dates = [ + release_date + for origin_match in self.origin_matches + if (release_date := get_release_date(origin_match)) + ] + return min(release_dates, default=None) + def is_same_package(self, other_origin): return bool(self.equivalent_purls & other_origin.equivalent_purls) @@ -233,37 +362,153 @@ def is_same_package_versions(origins): return bool(set.intersection(*(origin.versionless_purls for origin in origins))) -def get_rule(origins): - """Return the determination rule of the candidate ``origins`` of a resource.""" - if len(origins) > 1: - if is_same_package_versions(origins): - return CONFLICTING_VERSIONS_RULE - return CONFLICTING_CANDIDATES_RULE +def get_preferred_version_origin(origins): + """ + Return the origin of the preferred version among the ``origins`` versions of + the same package: the clearly most similar version, or else the only release + version, as opposed to a commit, or else the clearly most similar release, or + else the earliest release, as a file often stays the same in several releases. + Return None when no version is preferred. + """ + if most_similar_origin := get_most_similar_origin(origins): + return most_similar_origin - origin = origins[0] + release_origins = [origin for origin in origins if origin.is_release_version] + if len(release_origins) == 1: + return release_origins[0] + + # Both the similarity and the release date compare two releases at least. + if len(release_origins) < 2: + return + + if most_similar_release_origin := get_most_similar_origin(release_origins): + return most_similar_release_origin + + return get_earliest_release_origin(release_origins) + + +def get_earliest_release_origin(origins): + """ + Return the origin released before all the other ``origins`` versions of the + same package, as a file often stays the same in several releases and was + copied at the earliest. Return None when a release date is unknown, or when the + earliest date is shared. + """ + if not all(origin.release_date for origin in origins): + return + + earliest_origin, next_origin = sorted(origins, key=attrgetter("release_date"))[:2] + if earliest_origin.release_date < next_origin.release_date: + return earliest_origin + + +def get_most_similar_origin(origins): + """ + Return the origin more similar than all the other ``origins`` by at least + ``MIN_SIMILARITY_MARGIN`` points, as the copy resembles it the most. Return None + when a similarity is unknown, or when the margin is not reached. + """ + if not all(origin.similarity is not None for origin in origins): + return + + most_similar_origin, next_origin = sorted( + origins, key=attrgetter("similarity"), reverse=True + )[:2] + if most_similar_origin.similarity - next_origin.similarity >= MIN_SIMILARITY_MARGIN: + return most_similar_origin + + +def is_carrier_origin(origin, other_origins): + """ + Return True when the copy in the ``origin`` package is in a file, or in a + directory just under a vendor directory, named after one of the + ``other_origins`` packages, such as six.py vendored by pip in + "pip/_vendor/six.py": the ``origin`` carries the copy of another package. + A copy named after the ``origin`` package itself is its own file, such as + "six.py" in another version of six. + """ + carried_names = origin.copy_names - origin.package_names + return any( + carried_names & other_origin.package_names for other_origin in other_origins + ) + + +def get_non_carrier_origins(origins): + """Return the ``origins`` that do not carry the copy of another origin.""" + non_carrier_origins = [] + for origin in origins: + other_origins = [ + other_origin for other_origin in origins if other_origin is not origin + ] + if not is_carrier_origin(origin, other_origins): + non_carrier_origins.append(origin) + return non_carrier_origins + + +def get_conflict_rule_and_candidate(origins): + """ + Return the determination rule of the conflicting candidate ``origins`` of a + resource, and the proposed candidate origin, None when there is no proposal. + The candidates that carry the copy of another candidate are not its origin: + the other rules only compare the remaining candidates. + """ + non_carrier_origins = get_non_carrier_origins(origins) + if len(non_carrier_origins) == 1: + return CONTAINMENT_RULE, non_carrier_origins[0] + + # All the candidates are kept when they all carry the copy of another. + candidate_origins = non_carrier_origins or origins + + if is_same_package_versions(candidate_origins): + if preferred_origin := get_preferred_version_origin(candidate_origins): + return PREFERRED_VERSION_RULE, preferred_origin + return CONFLICTING_VERSIONS_RULE, None + + if most_similar_origin := get_most_similar_origin(candidate_origins): + return HIGHEST_SIMILARITY_RULE, most_similar_origin + + # The release date does not decide between different packages: the earliest + # one is often another copy, such as a GitHub repository copying the file, + # when the original package is not among the candidates. + return CONFLICTING_CANDIDATES_RULE, None + + +def get_single_origin_rule(origin): + """Return the determination rule of the only candidate ``origin`` of a resource.""" if not origin.has_exact_match: return NO_EXACT_MATCH_RULE - # A single tool can miss the other candidates: MatchCode.io stops at the - # first package matched, and SCANOSS reports one component per file. + # A single tool can miss the other candidates: SCANOSS reports one component + # per file, and MatchCode.io only knows the indexed packages. if len(origin.tools) < 2: return SINGLE_TOOL_RULE return CONFIRMED_BY_TWO_TOOLS_RULE +def get_rule_and_candidate(origins): + """ + Return the determination rule of the candidate ``origins`` of a resource, and + the proposed candidate origin, None when there is no proposal. + """ + if len(origins) > 1: + return get_conflict_rule_and_candidate(origins) + + origin = origins[0] + return get_single_origin_rule(origin), origin + + def make_origin_determination(project, resource, origins): """Return an unsaved OriginDetermination for the ``resource`` ``origins``.""" - rule = get_rule(origins) + rule, candidate_origin = get_rule_and_candidate(origins) status = OriginDetermination.Status.NEEDS_REVIEW if rule == CONFIRMED_BY_TWO_TOOLS_RULE: status = OriginDetermination.Status.AUTO_CONFIRMED - has_single_candidate = len(origins) == 1 return OriginDetermination( project=project, resource=resource, - purl=origins[0].purl if has_single_candidate else "", + purl=candidate_origin.purl if candidate_origin else "", status=status, rule=rule, ) @@ -382,16 +627,38 @@ def get_origin_packages(project): return project.discoveredpackages.filter(datasource_ids=[ORIGIN_DATASOURCE_ID]) +def get_scan_package(project, purl): + """ + Return the ``project`` package of ``purl`` detected by the codebase scan, or + else the same package version with other qualifiers or subpath, such as a + Maven package with a "sources" classifier, the one without qualifiers first. + Return None when there is none. + """ + purl_data = PackageURL.from_string(purl).to_dict() + lookup_purl_data = DiscoveredPackage.extract_purl_data(purl_data) + version_purl_data = { + field_name: value + for field_name, value in lookup_purl_data.items() + if field_name not in ["qualifiers", "subpath"] + } + same_version_packages = project.discoveredpackages.filter( + **version_purl_data + ).exclude(datasource_ids=[ORIGIN_DATASOURCE_ID]) + + if package := same_version_packages.filter(**lookup_purl_data).first(): + return package + return same_version_packages.order_by("qualifiers", "subpath").first() + + def get_or_create_origin_package(project, purl, resources): """ Return the ``project`` package of ``purl``, such as a package detected by the codebase scan, or else create it from the ``resources`` origin matches. """ - purl_data = PackageURL.from_string(purl).to_dict() - lookup_purl_data = DiscoveredPackage.extract_purl_data(purl_data) - if package := project.discoveredpackages.filter(**lookup_purl_data).first(): + if package := get_scan_package(project, purl): return package + purl_data = PackageURL.from_string(purl).to_dict() origin_matches = get_purl_origin_matches(project, purl, resources) package_data = { **purl_data, @@ -417,8 +684,13 @@ def assign_origin_package_resources(package, resources): return package.add_resources(new_resources) - new_resource_paths = [resource.path for resource in new_resources] - package.update_extra_data({ORIGIN_RESOURCE_PATHS_KEY: new_resource_paths}) + # Several origin PURLs can lead to the same scan package, such as a PURL with + # qualifiers confirmed by a user and the same PURL without qualifiers. + origin_resource_paths = [ + *package.extra_data.get(ORIGIN_RESOURCE_PATHS_KEY, []), + *(resource.path for resource in new_resources), + ] + package.update_extra_data({ORIGIN_RESOURCE_PATHS_KEY: origin_resource_paths}) def unassign_scan_package_resources(project): @@ -541,6 +813,8 @@ def determine_origins(project, batch_size=2000): "purl", "match_type", "tool", + "origin_path", + "similarity", "extra_data", "resource__path", "resource__type", diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 0b480162d6..5dd2691bf9 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -166,9 +166,12 @@ def collect_response_results(response, data, timeout=DEFAULT_TIMEOUT): next_page = response.get("next") while next_page: response = request_post(url=next_page, data=data, timeout=timeout) - if response and response.get("count"): - results.extend(response["results"]) - next_page = response.get("next") + # A failed request returns None, the same next page would be requested + # forever. + if not (response and response.get("count")): + break + results.extend(response["results"]) + next_page = response.get("next") return results diff --git a/scanpipe/tests/data/code-genetics/README.md b/scanpipe/tests/data/code-genetics/README.md index 6af89f3561..70d91fd155 100644 --- a/scanpipe/tests/data/code-genetics/README.md +++ b/scanpipe/tests/data/code-genetics/README.md @@ -10,6 +10,10 @@ Small synthetic codebases, one origin case each, to measure origin detection and | `snippet` | lodash `debounce` function copied into an application file | `pkg:npm/lodash@4.17.21` | | `embedded-package` | Complete ms npm package in a vendor directory | `pkg:npm/ms@2.1.3` | | `no-origin` | Original application code | None | +| `shared-copy` | Unmodified `six.py`, identical in six and in pip, which vendors it | `pkg:pypi/six@1.16.0` | +| `embedded-archive` | slf4j-nop Maven package archive embedded as a file | `pkg:maven/org.slf4j/slf4j-nop@2.0.16` | +| `single-tool-copy` | Unmodified escape-html `index.js`, not indexed in the local MatchCode.io | `pkg:npm/escape-html@1.0.3` | +| `license-header` | Original application code with the common MIT license header | None | ## Layout @@ -19,13 +23,14 @@ Each case directory contains: - `expected.json`: the ground truth. - `matchcode-results.json`: the MatchCode.io match results, reduced to the fields used by the importer. - `scanoss-results.json`: the SCANOSS raw results, with paths relative to `codebase/`. +- `purldb-resources.json`: the PurlDB resources of the files matched by MatchCode.io, reduced to their package `purl`, `sha1` and `path`, as returned by the `resources/filter_by_checksums/` API. Only for the `shared-copy` case, where the paths of the copies decide the origin. ## expected.json - `resources`: one entry per file of `codebase/`. - `path`: path relative to `codebase/`. - `match_type`: `exact`, `modified`, `snippet` or `none`, compared to the origin file. - - `origin`: the original file (`purl`, `download_url`, `path` inside the downloaded archive, `sha1`), or `null` when there is no origin. + - `origin`: the original file (`purl`, `download_url`, `path` inside the downloaded archive or the archive file name, `sha1`), or `null` when there is no origin. - `copied_from`: the file actually copied, when it differs from the origin (vendored copy of a vendored copy). - `line_ranges`: for `modified` and `snippet` only, the `lines` of the codebase file that come from the `origin_lines` of the origin file. Ranges are inclusive. - `packages`: complete packages embedded in the codebase, with their directory `path`. @@ -34,5 +39,5 @@ Each case directory contains: The tool results are real outputs, not handwritten: -- MatchCode.io results come from a local PurlDB indexing only the six reference packages of this data set (`pkg:pypi/six@1.16.0`, `pkg:pypi/six@1.17.0`, `pkg:pypi/urllib3@1.26.20`, `pkg:github/benhoyt/inih@r62`, `pkg:npm/lodash@4.17.21`, `pkg:npm/ms@2.1.3`), with the file and directory fingerprints computed by ScanCode.io. -- SCANOSS results come from the free osskb.org service (knowledge base 26.10.19), which reports a single best match per file. +- MatchCode.io results come from a local PurlDB indexing only the reference packages of this data set (`pkg:pypi/six@1.16.0`, `pkg:pypi/six@1.17.0`, `pkg:pypi/urllib3@1.26.20`, `pkg:github/benhoyt/inih@r62`, `pkg:npm/lodash@4.17.21`, `pkg:npm/ms@2.1.3`, `pkg:pypi/pip@24.0`, `pkg:maven/org.slf4j/slf4j-nop@2.0.16`), with the file and directory fingerprints computed by ScanCode.io. For the `shared-copy`, `embedded-archive`, `single-tool-copy` and `license-header` cases, six 1.16.0, pip and slf4j-nop are indexed with their archive `sha1` and registry release date. The snippets matched in legal files are removed, as the importer ignores them. +- SCANOSS results come from the free osskb.org service (knowledge base 26.10.19, 26.11.03 for the last four cases), which reports a single best match per file. diff --git a/scanpipe/tests/data/code-genetics/embedded-archive/codebase/jars/slf4j-nop-2.0.16.jar b/scanpipe/tests/data/code-genetics/embedded-archive/codebase/jars/slf4j-nop-2.0.16.jar new file mode 100644 index 0000000000..0fac10585d Binary files /dev/null and b/scanpipe/tests/data/code-genetics/embedded-archive/codebase/jars/slf4j-nop-2.0.16.jar differ diff --git a/scanpipe/tests/data/code-genetics/embedded-archive/expected.json b/scanpipe/tests/data/code-genetics/embedded-archive/expected.json new file mode 100644 index 0000000000..12d08a6ca3 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/embedded-archive/expected.json @@ -0,0 +1,15 @@ +{ + "description": "The slf4j-nop Maven package archive embedded as a file in an application lib directory.", + "resources": [ + { + "path": "jars/slf4j-nop-2.0.16.jar", + "match_type": "exact", + "origin": { + "purl": "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + "download_url": "https://repo1.maven.org/maven2/org/slf4j/slf4j-nop/2.0.16/slf4j-nop-2.0.16.jar", + "path": "slf4j-nop-2.0.16.jar", + "sha1": "d8d657e5b08dfbad70758fc8c47c58372272e08c" + } + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/embedded-archive/matchcode-results.json b/scanpipe/tests/data/code-genetics/embedded-archive/matchcode-results.json new file mode 100644 index 0000000000..c27ac6e705 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/embedded-archive/matchcode-results.json @@ -0,0 +1,39 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "tool_version": "38.1.0" + } + ], + "packages": [ + { + "purl": "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + "type": "maven", + "namespace": "org.slf4j", + "name": "slf4j-nop", + "version": "2.0.16", + "package_uid": "pkg:maven/org.slf4j/slf4j-nop@2.0.16?uuid=97c4844d-bb74-49bc-a117-c7a1be534254", + "release_date": "2024-08-10", + "declared_license_expression": "", + "download_url": "https://repo1.maven.org/maven2/org/slf4j/slf4j-nop/2.0.16/slf4j-nop-2.0.16.jar" + } + ], + "files": [ + { + "path": "jars", + "type": "directory", + "status": "", + "for_packages": [], + "extra_data": {} + }, + { + "path": "jars/slf4j-nop-2.0.16.jar", + "type": "file", + "status": "matched-to-purldb-package", + "for_packages": [ + "pkg:maven/org.slf4j/slf4j-nop@2.0.16?uuid=97c4844d-bb74-49bc-a117-c7a1be534254" + ], + "extra_data": {} + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/embedded-archive/scanoss-results.json b/scanpipe/tests/data/code-genetics/embedded-archive/scanoss-results.json new file mode 100644 index 0000000000..11921044c2 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/embedded-archive/scanoss-results.json @@ -0,0 +1,43 @@ +{ + "jars/slf4j-nop-2.0.16.jar": [ + { + "component": "slf4j-nop", + "file": "slf4j-nop-2.0.16.jar", + "file_hash": "619ab5f8e4a331b90530238faed7737f", + "file_url": " ", + "id": "file", + "latest": "2.0.16", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2026-10-22T15:26:00+0000", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:maven/org.slf4j/slf4j-nop" + ], + "release_date": "2024-08-10", + "server": { + "kb_version": { + "daily": "26.11.03", + "monthly": "26.10" + }, + "version": "5.4.28" + }, + "source_hash": "619ab5f8e4a331b90530238faed7737f", + "status": "pending", + "url": "https://mvnrepository.com/artifact/org.slf4j/slf4j-nop", + "url_hash": "619ab5f8e4a331b90530238faed7737f", + "vendor": "org.slf4j", + "version": "2.0.16" + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/license-header/codebase/src/sensor_calibration.py b/scanpipe/tests/data/code-genetics/license-header/codebase/src/sensor_calibration.py new file mode 100644 index 0000000000..6cf0a400e1 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/license-header/codebase/src/sensor_calibration.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026 Greenhouse Collective +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Calibration of the soil moisture sensors of the greenhouse benches.""" + +from dataclasses import dataclass + + +@dataclass +class SensorReading: + sensor_id: str + raw_value: int + + +@dataclass +class Calibration: + dry_value: int + wet_value: int + + def to_moisture_percent(self, raw_value): + span = self.dry_value - self.wet_value + if span <= 0: + raise ValueError("The dry value must be higher than the wet value.") + + clamped_value = min(max(raw_value, self.wet_value), self.dry_value) + return round(100 * (self.dry_value - clamped_value) / span) + + +def calibrate_readings(readings, calibration_by_sensor_id): + moisture_by_sensor_id = {} + for reading in readings: + calibration = calibration_by_sensor_id.get(reading.sensor_id) + if not calibration: + continue + + moisture_by_sensor_id[reading.sensor_id] = calibration.to_moisture_percent( + reading.raw_value + ) + return moisture_by_sensor_id + + +def driest_sensor(moisture_by_sensor_id): + if not moisture_by_sensor_id: + return + + return min(moisture_by_sensor_id, key=moisture_by_sensor_id.get) diff --git a/scanpipe/tests/data/code-genetics/license-header/expected.json b/scanpipe/tests/data/code-genetics/license-header/expected.json new file mode 100644 index 0000000000..27cff6ebef --- /dev/null +++ b/scanpipe/tests/data/code-genetics/license-header/expected.json @@ -0,0 +1,10 @@ +{ + "description": "Original application code with the common MIT license header. Any match is a false positive.", + "resources": [ + { + "path": "src/sensor_calibration.py", + "match_type": "none", + "origin": null + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/license-header/matchcode-results.json b/scanpipe/tests/data/code-genetics/license-header/matchcode-results.json new file mode 100644 index 0000000000..13f40da453 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/license-header/matchcode-results.json @@ -0,0 +1,1362 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "tool_version": "38.1.0" + } + ], + "packages": [ + { + "purl": "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + "type": "maven", + "namespace": "org.slf4j", + "name": "slf4j-nop", + "version": "2.0.16", + "package_uid": "pkg:maven/org.slf4j/slf4j-nop@2.0.16?uuid=97c4844d-bb74-49bc-a117-c7a1be534254", + "release_date": "2024-08-10", + "declared_license_expression": "", + "download_url": "https://repo1.maven.org/maven2/org/slf4j/slf4j-nop/2.0.16/slf4j-nop-2.0.16.jar" + }, + { + "purl": "pkg:npm/lodash@4.17.21", + "type": "npm", + "namespace": "", + "name": "lodash", + "version": "4.17.21", + "package_uid": "pkg:npm/lodash@4.17.21?uuid=e3ceae14-876f-4659-8cf3-836122f64823", + "release_date": null, + "declared_license_expression": "", + "download_url": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + }, + { + "purl": "pkg:npm/ms@2.1.3", + "type": "npm", + "namespace": "", + "name": "ms", + "version": "2.1.3", + "package_uid": "pkg:npm/ms@2.1.3?uuid=59bbd3a1-0d87-4783-9bc3-41c2d399c473", + "release_date": null, + "declared_license_expression": "", + "download_url": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + }, + { + "purl": "pkg:pypi/pip@24.0", + "type": "pypi", + "namespace": "", + "name": "pip", + "version": "24.0", + "package_uid": "pkg:pypi/pip@24.0?uuid=9b42306f-61c6-48e3-81a3-966eba6a24c3", + "release_date": "2024-02-03", + "declared_license_expression": "", + "download_url": "https://files.pythonhosted.org/packages/source/p/pip/pip-24.0.tar.gz" + }, + { + "purl": "pkg:pypi/six@1.16.0", + "type": "pypi", + "namespace": "", + "name": "six", + "version": "1.16.0", + "package_uid": "pkg:pypi/six@1.16.0?uuid=c23c7e08-8906-4468-973b-333a7ab91cf7", + "release_date": "2021-05-05", + "declared_license_expression": "", + "download_url": "https://files.pythonhosted.org/packages/source/s/six/six-1.16.0.tar.gz" + }, + { + "purl": "pkg:pypi/six@1.17.0", + "type": "pypi", + "namespace": "", + "name": "six", + "version": "1.17.0", + "package_uid": "pkg:pypi/six@1.17.0?uuid=b25b1d26-be0c-49aa-b04d-55f529612878", + "release_date": null, + "declared_license_expression": "", + "download_url": "https://files.pythonhosted.org/packages/source/s/six/six-1.17.0.tar.gz" + }, + { + "purl": "pkg:pypi/urllib3@1.26.20", + "type": "pypi", + "namespace": "", + "name": "urllib3", + "version": "1.26.20", + "package_uid": "pkg:pypi/urllib3@1.26.20?uuid=26448037-1ab6-4b6d-847e-a7c079e13590", + "release_date": null, + "declared_license_expression": "", + "download_url": "https://files.pythonhosted.org/packages/source/u/urllib3/urllib3-1.26.20.tar.gz" + } + ], + "files": [ + { + "path": "src", + "type": "directory", + "status": "", + "for_packages": [], + "extra_data": {} + }, + { + "path": "src/sensor_calibration.py", + "type": "file", + "status": "", + "for_packages": [ + "pkg:maven/org.slf4j/slf4j-nop@2.0.16?uuid=97c4844d-bb74-49bc-a117-c7a1be534254", + "pkg:npm/ms@2.1.3?uuid=59bbd3a1-0d87-4783-9bc3-41c2d399c473", + "pkg:npm/lodash@4.17.21?uuid=e3ceae14-876f-4659-8cf3-836122f64823", + "pkg:pypi/urllib3@1.26.20?uuid=26448037-1ab6-4b6d-847e-a7c079e13590", + "pkg:pypi/pip@24.0?uuid=9b42306f-61c6-48e3-81a3-966eba6a24c3", + "pkg:pypi/six@1.16.0?uuid=c23c7e08-8906-4468-973b-333a7ab91cf7", + "pkg:pypi/six@1.17.0?uuid=b25b1d26-be0c-49aa-b04d-55f529612878" + ], + "extra_data": { + "matched_snippets": [ + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/setup.py", + "similarity": 0.26666666666666666, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/setup.py", + "similarity": 0.19047619047619047, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_internal/utils/_jaraco_text.py", + "similarity": 0.15873015873015872, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/dummyserver/proxy.py", + "similarity": 0.14388489208633093, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/rich/_spinners.py", + "similarity": 0.11049723756906077, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/pyparsing/__init__.py", + "similarity": 0.055710306406685235, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.055710306406685235, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/_securetransport/bindings.py", + "similarity": 0.03067484662576687, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/contrib/securetransport.py", + "similarity": 0.03067484662576687, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/contrib/securetransport.py", + "similarity": 0.02976190476190476, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/test_six.py", + "similarity": 0.02936857562408223, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/test_six.py", + "similarity": 0.02717391304347826, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159] + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py", + "similarity": 0.02717391304347826, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.02717391304347826, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/urllib3/packages/six.py", + "similarity": 0.02717391304347826, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/urllib3@1.26.20", + "resource": "urllib3-1.26.20.tar.gz-extract/urllib3-1.26.20/src/urllib3/packages/six.py", + "similarity": 0.02706359945872801, + "match_detections": [152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/six@1.17.0", + "resource": "six-1.17.0.tar.gz-extract/six-1.17.0/six.py", + "similarity": 0.005656108597285068, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/pyparsing/core.py", + "similarity": 0.005656108597285068, + "match_detections": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167] + } + ] + } + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/license-header/scanoss-results.json b/scanpipe/tests/data/code-genetics/license-header/scanoss-results.json new file mode 100644 index 0000000000..067331cffa --- /dev/null +++ b/scanpipe/tests/data/code-genetics/license-header/scanoss-results.json @@ -0,0 +1,14 @@ +{ + "src/sensor_calibration.py": [ + { + "id": "none", + "server": { + "kb_version": { + "daily": "26.11.03", + "monthly": "26.10" + }, + "version": "5.4.28" + } + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/shared-copy/codebase/third_party/six.py b/scanpipe/tests/data/code-genetics/shared-copy/codebase/third_party/six.py new file mode 100644 index 0000000000..4e15675d8b --- /dev/null +++ b/scanpipe/tests/data/code-genetics/shared-copy/codebase/third_party/six.py @@ -0,0 +1,998 @@ +# Copyright (c) 2010-2020 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Utilities for writing code that runs on Python 2 and 3""" + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.16.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + del io + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") + + +if sys.version_info[:2] > (3,): + exec_("""def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(type): + + def __new__(cls, name, this_bases, d): + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, binary_type): + return s + if isinstance(s, text_type): + return s.encode(encoding, errors) + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + # Optimization: Fast return for the common case. + if type(s) is str: + return s + if PY2 and isinstance(s, text_type): + return s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def python_2_unicode_compatible(klass): + """ + A class decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/scanpipe/tests/data/code-genetics/shared-copy/expected.json b/scanpipe/tests/data/code-genetics/shared-copy/expected.json new file mode 100644 index 0000000000..cea3e06a28 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/shared-copy/expected.json @@ -0,0 +1,15 @@ +{ + "description": "Unmodified six.py, identical in six and in pip, which vendors it. The origin is six, not pip.", + "resources": [ + { + "path": "third_party/six.py", + "match_type": "exact", + "origin": { + "purl": "pkg:pypi/six@1.16.0", + "download_url": "https://files.pythonhosted.org/packages/source/s/six/six-1.16.0.tar.gz", + "path": "six-1.16.0/six.py", + "sha1": "d2b72496fefbd26201ecc94881e42bb0ac6e3374" + } + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/shared-copy/matchcode-results.json b/scanpipe/tests/data/code-genetics/shared-copy/matchcode-results.json new file mode 100644 index 0000000000..5d0fe788d4 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/shared-copy/matchcode-results.json @@ -0,0 +1,51 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "tool_version": "38.1.0" + } + ], + "packages": [ + { + "purl": "pkg:pypi/pip@24.0", + "type": "pypi", + "namespace": "", + "name": "pip", + "version": "24.0", + "package_uid": "pkg:pypi/pip@24.0?uuid=9b42306f-61c6-48e3-81a3-966eba6a24c3", + "release_date": "2024-02-03", + "declared_license_expression": "", + "download_url": "https://files.pythonhosted.org/packages/source/p/pip/pip-24.0.tar.gz" + }, + { + "purl": "pkg:pypi/six@1.16.0", + "type": "pypi", + "namespace": "", + "name": "six", + "version": "1.16.0", + "package_uid": "pkg:pypi/six@1.16.0?uuid=c23c7e08-8906-4468-973b-333a7ab91cf7", + "release_date": "2021-05-05", + "declared_license_expression": "", + "download_url": "https://files.pythonhosted.org/packages/source/s/six/six-1.16.0.tar.gz" + } + ], + "files": [ + { + "path": "third_party", + "type": "directory", + "status": "", + "for_packages": [], + "extra_data": {} + }, + { + "path": "third_party/six.py", + "type": "file", + "status": "matched-to-purldb-resource", + "for_packages": [ + "pkg:pypi/pip@24.0?uuid=9b42306f-61c6-48e3-81a3-966eba6a24c3", + "pkg:pypi/six@1.16.0?uuid=c23c7e08-8906-4468-973b-333a7ab91cf7" + ], + "extra_data": {} + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/shared-copy/purldb-resources.json b/scanpipe/tests/data/code-genetics/shared-copy/purldb-resources.json new file mode 100644 index 0000000000..606ebe701c --- /dev/null +++ b/scanpipe/tests/data/code-genetics/shared-copy/purldb-resources.json @@ -0,0 +1,12 @@ +[ + { + "purl": "pkg:pypi/six@1.16.0", + "sha1": "d2b72496fefbd26201ecc94881e42bb0ac6e3374", + "path": "six-1.16.0.tar.gz-extract/six-1.16.0/six.py" + }, + { + "purl": "pkg:pypi/pip@24.0", + "sha1": "d2b72496fefbd26201ecc94881e42bb0ac6e3374", + "path": "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py" + } +] diff --git a/scanpipe/tests/data/code-genetics/shared-copy/scanoss-results.json b/scanpipe/tests/data/code-genetics/shared-copy/scanoss-results.json new file mode 100644 index 0000000000..a0d042d270 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/shared-copy/scanoss-results.json @@ -0,0 +1,46 @@ +{ + "third_party/six.py": [ + { + "component": "six", + "file": "six.py", + "file_hash": "9379cf68c692d9a9f92e5d29f6a54549", + "file_url": " ", + "id": "file", + "latest": "1.16.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2026-10-22T15:26:00+0000", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/benjaminp/six", + "pkg:pypi/six", + "pkg:golang/github.com/benjaminp/six", + "pkg:conda/six" + ], + "release_date": "2021-05-05", + "server": { + "kb_version": { + "daily": "26.11.03", + "monthly": "26.10" + }, + "version": "5.4.28" + }, + "source_hash": "9379cf68c692d9a9f92e5d29f6a54549", + "status": "pending", + "url": "https://github.com/benjaminp/six", + "url_hash": "3d21479ad1558562f8ea4ff43a1c0c72", + "vendor": "benjaminp", + "version": "1.16.0" + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/single-tool-copy/codebase/static/js/escape-html.js b/scanpipe/tests/data/code-genetics/single-tool-copy/codebase/static/js/escape-html.js new file mode 100644 index 0000000000..bf9e226f4e --- /dev/null +++ b/scanpipe/tests/data/code-genetics/single-tool-copy/codebase/static/js/escape-html.js @@ -0,0 +1,78 @@ +/*! + * escape-html + * Copyright(c) 2012-2013 TJ Holowaychuk + * Copyright(c) 2015 Andreas Lubbe + * Copyright(c) 2015 Tiancheng "Timothy" Gu + * MIT Licensed + */ + +'use strict'; + +/** + * Module variables. + * @private + */ + +var matchHtmlRegExp = /["'&<>]/; + +/** + * Module exports. + * @public + */ + +module.exports = escapeHtml; + +/** + * Escape special characters in the given string of html. + * + * @param {string} string The string to escape for inserting into HTML + * @return {string} + * @public + */ + +function escapeHtml(string) { + var str = '' + string; + var match = matchHtmlRegExp.exec(str); + + if (!match) { + return str; + } + + var escape; + var html = ''; + var index = 0; + var lastIndex = 0; + + for (index = match.index; index < str.length; index++) { + switch (str.charCodeAt(index)) { + case 34: // " + escape = '"'; + break; + case 38: // & + escape = '&'; + break; + case 39: // ' + escape = '''; + break; + case 60: // < + escape = '<'; + break; + case 62: // > + escape = '>'; + break; + default: + continue; + } + + if (lastIndex !== index) { + html += str.substring(lastIndex, index); + } + + lastIndex = index + 1; + html += escape; + } + + return lastIndex !== index + ? html + str.substring(lastIndex, index) + : html; +} diff --git a/scanpipe/tests/data/code-genetics/single-tool-copy/expected.json b/scanpipe/tests/data/code-genetics/single-tool-copy/expected.json new file mode 100644 index 0000000000..f78e05dd34 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/single-tool-copy/expected.json @@ -0,0 +1,15 @@ +{ + "description": "Unmodified escape-html index.js copied in an application. Its package is not indexed in the local MatchCode.io: only SCANOSS can report it.", + "resources": [ + { + "path": "static/js/escape-html.js", + "match_type": "exact", + "origin": { + "purl": "pkg:npm/escape-html@1.0.3", + "download_url": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "path": "package/index.js", + "sha1": "dfbb19c79eb0ca7ff2625fb1975a35cf47be378a" + } + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/single-tool-copy/matchcode-results.json b/scanpipe/tests/data/code-genetics/single-tool-copy/matchcode-results.json new file mode 100644 index 0000000000..e12c4e3d01 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/single-tool-copy/matchcode-results.json @@ -0,0 +1,32 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "tool_version": "38.1.0" + } + ], + "packages": [], + "files": [ + { + "path": "static", + "type": "directory", + "status": "", + "for_packages": [], + "extra_data": {} + }, + { + "path": "static/js", + "type": "directory", + "status": "", + "for_packages": [], + "extra_data": {} + }, + { + "path": "static/js/escape-html.js", + "type": "file", + "status": "", + "for_packages": [], + "extra_data": {} + } + ] +} diff --git a/scanpipe/tests/data/code-genetics/single-tool-copy/scanoss-results.json b/scanpipe/tests/data/code-genetics/single-tool-copy/scanoss-results.json new file mode 100644 index 0000000000..44f85ad3c2 --- /dev/null +++ b/scanpipe/tests/data/code-genetics/single-tool-copy/scanoss-results.json @@ -0,0 +1,45 @@ +{ + "static/js/escape-html.js": [ + { + "component": "escape-html", + "file": "index.js", + "file_hash": "0c95e46d0f08bd96b93cfbea66888afc", + "file_url": " ", + "id": "file", + "latest": "v1.0.3", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2026-10-22T15:26:00+0000", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/component/escape-html", + "pkg:maven/org.webjars.npm/escape-html", + "pkg:npm/%40samlfair/escape-html" + ], + "release_date": "2015-09-01", + "server": { + "kb_version": { + "daily": "26.11.03", + "monthly": "26.10" + }, + "version": "5.4.28" + }, + "source_hash": "0c95e46d0f08bd96b93cfbea66888afc", + "status": "pending", + "url": "https://github.com/component/escape-html", + "url_hash": "30d7c749d83f6664a68a1a3a85f3a122", + "vendor": "component", + "version": "v1.0.3" + } + ] +} diff --git a/scanpipe/tests/pipes/test_matchcode.py b/scanpipe/tests/pipes/test_matchcode.py index b18175f3fa..9790155154 100644 --- a/scanpipe/tests/pipes/test_matchcode.py +++ b/scanpipe/tests/pipes/test_matchcode.py @@ -37,6 +37,8 @@ from scanpipe.tests import make_resource_directory from scanpipe.tests import make_resource_file +SIX_16_SHA1 = "d2b72496fefbd26201ecc94881e42bb0ac6e3374" + class MatchCodePipesTest(TestCase): data = Path(__file__).parent.parent / "data" @@ -532,6 +534,46 @@ def test_scanpipe_pipes_matchcode_get_origin_match_extra_data(self): } self.assertEqual(expected, matchcode.get_origin_match_extra_data(package_data)) + def test_scanpipe_pipes_matchcode_merge_matched_snippets(self): + matched_snippets = [ + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0/six.py", + "similarity": 0.36, + "match_detections": [9, 10, 11], + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.36, + "match_detections": [9, 10, 11], + }, + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0/six.py", + "similarity": 0.38, + "match_detections": [11, 12, 13], + }, + ] + + expected = [ + { + "package": "pkg:pypi/six@1.16.0", + "resource": "six-1.16.0/six.py", + "similarity": 0.38, + "match_detections": [9, 10, 11, 12, 13], + }, + { + "package": "pkg:pypi/pip@24.0", + "resource": "pip-24.0/src/pip/_vendor/six.py", + "similarity": 0.36, + "match_detections": [9, 10, 11], + }, + ] + self.assertEqual(expected, matchcode.merge_matched_snippets(matched_snippets)) + # The input snippets are not modified. + self.assertEqual([9, 10, 11], matched_snippets[0]["match_detections"]) + def test_scanpipe_pipes_matchcode_make_snippet_origin_matches(self): resource = self.make_scheduler_resource("scheduler.js") matched_snippets = [ @@ -561,6 +603,38 @@ def test_scanpipe_pipes_matchcode_make_snippet_origin_matches(self): self.assertEqual(0.005, origin_match.extra_data["snippet_similarity"]) self.assertEqual("38.1.0", origin_match.tool_version) + def test_scanpipe_pipes_matchcode_make_snippet_origin_matches_same_origin_file( + self, + ): + # Several snippets of the same origin file make a single origin match, as + # the origin matches are unique by origin file. + resource = self.make_scheduler_resource("scheduler.js") + matched_snippets = [ + { + "package": "pkg:npm/lodash@4.17.21", + "resource": "package/debounce.js", + "similarity": 0.005, + "match_detections": [59, 60, 61], + }, + { + "package": "pkg:npm/lodash@4.17.21", + "resource": "package/debounce.js", + "similarity": 0.006, + "match_detections": [300, 301, 302], + }, + ] + + origin_matches = list( + matchcode.make_snippet_origin_matches( + resource, matched_snippets, {}, "38.1.0" + ) + ) + self.assertEqual(1, len(origin_matches)) + origin_match = origin_matches[0] + expected = [{"lines": [26, 28]}, {"lines": [111, 114]}] + self.assertEqual(expected, origin_match.line_ranges) + self.assertEqual(0.006, origin_match.extra_data["snippet_similarity"]) + def test_scanpipe_pipes_matchcode_make_package_origin_matches(self): resource = make_resource_file(self.project1, "index.js") package_data_by_uid = { @@ -575,6 +649,7 @@ def test_scanpipe_pipes_matchcode_make_package_origin_matches(self): ["uid1", "uid2"], package_data_by_uid, "38.1.0", + origin_paths={}, ) ) self.assertEqual(1, len(origin_matches)) @@ -588,9 +663,134 @@ def test_scanpipe_pipes_matchcode_make_package_origin_matches(self): ["uid1"], package_data_by_uid, "38.1.0", + origin_paths={}, ) ) self.assertEqual(100, origin_matches[0].similarity) + self.assertEqual("", origin_matches[0].origin_path) + + def test_scanpipe_pipes_matchcode_make_package_origin_matches_origin_paths(self): + resource = make_resource_file(self.project1, "six.py", sha1=SIX_16_SHA1) + package_data_by_uid = { + "uid1": {"purl": "pkg:pypi/six@1.16.0", "package_uid": "uid1"}, + "uid2": {"purl": "pkg:pypi/pip@24.0", "package_uid": "uid2"}, + } + origin_paths = { + ("pkg:pypi/pip@24.0", SIX_16_SHA1): "pip-24.0/src/pip/_vendor/six.py", + } + + origin_matches = matchcode.make_package_origin_matches( + resource, + OriginMatch.MatchType.FILE, + ["uid1", "uid2"], + package_data_by_uid, + "38.1.0", + origin_paths, + ) + values = [ + (origin_match.purl, origin_match.origin_path) + for origin_match in origin_matches + ] + expected = [ + ("pkg:pypi/six@1.16.0", ""), + ("pkg:pypi/pip@24.0", "pip-24.0/src/pip/_vendor/six.py"), + ] + self.assertEqual(expected, values) + + def test_scanpipe_pipes_matchcode_get_path_depth_and_path(self): + purldb_resource = {"path": "pip-24.0/src/pip/_vendor/six.py"} + expected = (4, "pip-24.0/src/pip/_vendor/six.py") + self.assertEqual(expected, matchcode.get_path_depth_and_path(purldb_resource)) + + @mock.patch("scanpipe.pipes.purldb.match_resources") + @mock.patch("scanpipe.pipes.purldb.is_configured") + def test_scanpipe_pipes_matchcode_get_origin_paths( + self, mock_is_configured, mock_match_resources + ): + candidate_purls = {"pkg:pypi/six@1.16.0", "pkg:pypi/pip@24.0"} + mock_is_configured.return_value = False + origin_paths = matchcode.get_origin_paths([SIX_16_SHA1], candidate_purls) + self.assertEqual({}, origin_paths) + mock_match_resources.assert_not_called() + + mock_is_configured.return_value = True + self.assertEqual({}, matchcode.get_origin_paths([], candidate_purls)) + mock_match_resources.assert_not_called() + + shared_copy_resources_location = ( + self.data / "code-genetics" / "shared-copy" / "purldb-resources.json" + ) + purldb_resources = json.loads(shared_copy_resources_location.read_text()) + # A second copy of the file in the pip package, deeper. + purldb_resources.append( + { + "purl": "pkg:pypi/pip@24.0", + "sha1": SIX_16_SHA1, + "path": "pip-24.0.tar.gz-extract/pip-24.0/tests/data/pkg/lib/six.py", + } + ) + # A copy of the file in a package that is not a candidate. + purldb_resources.append( + { + "purl": "pkg:pypi/virtualenv@20.0.0", + "sha1": SIX_16_SHA1, + "path": "virtualenv-20.0.0/src/virtualenv/six.py", + } + ) + mock_match_resources.return_value = purldb_resources + + expected = { + ("pkg:pypi/six@1.16.0", SIX_16_SHA1): ( + "six-1.16.0.tar.gz-extract/six-1.16.0/six.py" + ), + ("pkg:pypi/pip@24.0", SIX_16_SHA1): ( + "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py" + ), + } + origin_paths = matchcode.get_origin_paths([SIX_16_SHA1], candidate_purls) + self.assertEqual(expected, origin_paths) + mock_match_resources.assert_called_once_with([SIX_16_SHA1]) + + @mock.patch("scanpipe.pipes.purldb.match_resources") + @mock.patch("scanpipe.pipes.purldb.is_configured") + def test_scanpipe_pipes_matchcode_get_origin_paths_batches( + self, mock_is_configured, mock_match_resources + ): + mock_is_configured.return_value = True + mock_match_resources.side_effect = [ + [{"purl": "pkg:pypi/six@1.16.0", "sha1": "sha1_a", "path": "six.py"}], + [{"purl": "pkg:npm/ms@2.1.3", "sha1": "sha1_c", "path": "index.js"}], + ] + + origin_paths = matchcode.get_origin_paths( + ["sha1_a", "sha1_b", "sha1_c"], + {"pkg:pypi/six@1.16.0", "pkg:npm/ms@2.1.3"}, + batch_size=2, + ) + expected = { + ("pkg:pypi/six@1.16.0", "sha1_a"): "six.py", + ("pkg:npm/ms@2.1.3", "sha1_c"): "index.js", + } + self.assertEqual(expected, origin_paths) + expected_calls = [mock.call(["sha1_a", "sha1_b"]), mock.call(["sha1_c"])] + self.assertEqual(expected_calls, mock_match_resources.call_args_list) + + def test_scanpipe_pipes_matchcode_get_matched_file_sha1s(self): + make_resource_file(self.project1, "six.py", sha1=SIX_16_SHA1) + make_resource_file(self.project1, "lib/six.py", sha1=SIX_16_SHA1) + make_resource_file(self.project1, "index.js", sha1="ea869663") + make_resource_file(self.project1, "empty.py") + make_resource_file(self.project1, "LICENSE", sha1="2b8b815229aa8a61") + results_by_path = { + "six.py": {"status": "matched-to-purldb-resource"}, + "lib/six.py": {"status": "matched-to-purldb-resource"}, + "index.js": {"status": "matched-to-purldb-directory"}, + "empty.py": {"status": "matched-to-purldb-resource"}, + "LICENSE": {"status": "matched-to-purldb-resource"}, + } + + sha1s = matchcode.get_matched_file_sha1s(self.project1, results_by_path) + self.assertEqual([SIX_16_SHA1], sha1s) def test_scanpipe_pipes_matchcode_make_origin_matches(self): resource = self.make_scheduler_resource("scheduler.js") @@ -610,6 +810,7 @@ def test_scanpipe_pipes_matchcode_make_origin_matches(self): package_data_by_uid, package_data_by_purl, "38.1.0", + origin_paths={}, ) ) self.assertEqual(OriginMatch.MatchType.FILE, origin_matches[0].match_type) @@ -635,6 +836,7 @@ def test_scanpipe_pipes_matchcode_make_origin_matches(self): package_data_by_uid, package_data_by_purl, "38.1.0", + origin_paths={}, ) ) self.assertEqual(OriginMatch.MatchType.SNIPPET, origin_matches[0].match_type) @@ -646,6 +848,7 @@ def test_scanpipe_pipes_matchcode_make_origin_matches(self): package_data_by_uid, package_data_by_purl, "38.1.0", + origin_paths={}, ) self.assertEqual([], list(origin_matches)) @@ -685,3 +888,33 @@ def test_scanpipe_pipes_matchcode_load_matchcode_results(self): self.assertEqual([{"lines": [26, 140]}], snippet_origin_match.line_ranges) self.assertEqual("matchcode", snippet_origin_match.tool) self.assertEqual("38.1.0", snippet_origin_match.tool_version) + + @mock.patch("scanpipe.pipes.purldb.match_resources") + @mock.patch("scanpipe.pipes.purldb.is_configured") + def test_scanpipe_pipes_matchcode_load_matchcode_results_origin_paths( + self, mock_is_configured, mock_match_resources + ): + case_path = self.data / "code-genetics" / "shared-copy" + match_results = json.loads((case_path / "matchcode-results.json").read_text()) + purldb_resources_location = case_path / "purldb-resources.json" + mock_is_configured.return_value = True + mock_match_resources.return_value = json.loads( + purldb_resources_location.read_text() + ) + make_resource_directory(self.project1, "third_party") + make_resource_file(self.project1, "third_party/six.py", sha1=SIX_16_SHA1) + + matchcode.load_matchcode_results(self.project1, match_results) + mock_match_resources.assert_called_once_with([SIX_16_SHA1]) + + origin_matches = self.project1.originmatchs.order_by("purl") + expected = [ + ( + "pkg:pypi/pip@24.0", + "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py", + ), + ("pkg:pypi/six@1.16.0", "six-1.16.0.tar.gz-extract/six-1.16.0/six.py"), + ] + self.assertEqual( + expected, list(origin_matches.values_list("purl", "origin_path")) + ) diff --git a/scanpipe/tests/pipes/test_origin.py b/scanpipe/tests/pipes/test_origin.py index 26b7aec208..f91f12b6e8 100644 --- a/scanpipe/tests/pipes/test_origin.py +++ b/scanpipe/tests/pipes/test_origin.py @@ -22,7 +22,9 @@ import json import shutil +from datetime import date from pathlib import Path +from unittest import mock from django.test import TestCase @@ -31,6 +33,7 @@ from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes import matchcode from scanpipe.pipes import origin +from scanpipe.pipes import purldb from scanpipe.pipes import scanoss from scanpipe.tests import make_package from scanpipe.tests import make_project @@ -40,15 +43,40 @@ SCANOSS_SIX_PURLS = ["pkg:github/benjaminp/six", "pkg:pypi/six", "pkg:conda/six"] -def make_unsaved_origin_match(purl, tool="matchcode", match_type="file", purls=None): +def make_unsaved_origin_match( + purl, + tool="matchcode", + match_type="file", + purls=None, + similarity=None, + origin_path="", + release_date="", +): extra_data = {"purls": purls} if purls else {} + if release_date: + extra_data["release_date"] = release_date return OriginMatch( - purl=purl, tool=tool, match_type=match_type, extra_data=extra_data + purl=purl, + tool=tool, + match_type=match_type, + origin_path=origin_path, + similarity=similarity, + extra_data=extra_data, ) -def make_origin(purl, tool="matchcode", match_type="file", purls=None): - origin_match = make_unsaved_origin_match(purl, tool, match_type, purls) +def make_origin( + purl, + tool="matchcode", + match_type="file", + purls=None, + similarity=None, + origin_path="", + release_date="", +): + origin_match = make_unsaved_origin_match( + purl, tool, match_type, purls, similarity, origin_path, release_date + ) return origin.Origin.from_origin_match(origin_match) @@ -73,7 +101,13 @@ def setUp(self): self.project1 = make_project() def make_origin_match( - self, resource, purl, tool="matchcode", match_type="file", extra_data=None + self, + resource, + purl, + tool="matchcode", + match_type="file", + extra_data=None, + line_ranges=None, ): return OriginMatch.objects.create( project=self.project1, @@ -82,8 +116,17 @@ def make_origin_match( match_type=match_type, tool=tool, extra_data=extra_data or {}, + line_ranges=line_ranges or [], ) + @staticmethod + def get_purldb_resources(case_path): + """Return the PurlDB resources of the files matched by MatchCode.io.""" + resources_location = case_path / "purldb-resources.json" + if not resources_location.exists(): + return [] + return json.loads(resources_location.read_text()) + def load_data_set_case(self, case_name, tools=None): """ Return a project with the codebase and the results of the ``tools`` of a @@ -100,7 +143,14 @@ def load_data_set_case(self, case_name, tools=None): if matchcode.TOOL_NAME in tools: results_location = case_path / "matchcode-results.json" match_results = json.loads(results_location.read_text()) - matchcode.load_matchcode_results(project, match_results) + purldb_resources = self.get_purldb_resources(case_path) + with ( + mock.patch.object(purldb, "is_configured", return_value=True), + mock.patch.object( + purldb, "match_resources", return_value=purldb_resources + ), + ): + matchcode.load_matchcode_results(project, match_results) if scanoss.TOOL_NAME in tools: results_location = case_path / "scanoss-results.json" @@ -150,6 +200,70 @@ def test_scanpipe_pipes_origin_get_equivalent_purls(self): expected = {"pkg:pypi/six@1.17.0"} self.assertEqual(expected, origin.get_equivalent_purls(origin_match)) + origin_match = make_unsaved_origin_match( + "pkg:github/component/escape-html@v1.0.3", + tool="scanoss", + purls=["pkg:npm/escape-html"], + ) + expected = { + "pkg:github/component/escape-html@v1.0.3", + "pkg:github/component/escape-html@1.0.3", + "pkg:npm/escape-html@v1.0.3", + "pkg:npm/escape-html@1.0.3", + } + self.assertEqual(expected, origin.get_equivalent_purls(origin_match)) + + def test_scanpipe_pipes_origin_get_equivalent_versions(self): + self.assertEqual(["v1.0.3", "1.0.3"], origin.get_equivalent_versions("v1.0.3")) + self.assertEqual(["1.0.3"], origin.get_equivalent_versions("1.0.3")) + self.assertEqual(["vanilla"], origin.get_equivalent_versions("vanilla")) + self.assertEqual([None], origin.get_equivalent_versions(None)) + + def test_scanpipe_pipes_origin_is_release_version(self): + self.assertFalse(origin.is_release_version("498f34b")) + self.assertFalse( + origin.is_release_version("498f34b4c6b5d8a0e3f1a2b7c9d0e1f2a3b4c5d6") + ) + self.assertFalse(origin.is_release_version(None)) + self.assertFalse(origin.is_release_version("")) + self.assertTrue(origin.is_release_version("r62")) + self.assertTrue(origin.is_release_version("1.17.0")) + self.assertTrue(origin.is_release_version("v1.0.3")) + self.assertTrue(origin.is_release_version("20240101")) + self.assertTrue(origin.is_release_version("abc12")) + + def test_scanpipe_pipes_origin_normalize_name(self): + self.assertEqual( + "typing_extensions", origin.normalize_name("Typing-Extensions") + ) + + def test_scanpipe_pipes_origin_get_copy_names(self): + path = "pip-24.0.tar.gz-extract/pip-24.0/src/pip/_vendor/six.py" + self.assertEqual({"six"}, origin.get_copy_names(path)) + self.assertEqual({"six"}, origin.get_copy_names("six.py")) + + path = "pip-24.0/src/pip/_vendor/requests/models.py" + self.assertEqual({"requests", "models"}, origin.get_copy_names(path)) + + # The other directories are not named after a package, such as "utils". + path = "foo-1.0/src/utils/index.js" + self.assertEqual({"index"}, origin.get_copy_names(path)) + + expected = {"jquery.min", "jquery"} + self.assertEqual(expected, origin.get_copy_names("vendor/jquery.min.js")) + expected = {"socket.io", "socket"} + self.assertEqual(expected, origin.get_copy_names("socket.io.js")) + + def test_scanpipe_pipes_origin_get_release_date(self): + origin_match = make_unsaved_origin_match("pkg:pypi/six@1.16.0") + self.assertIsNone(origin.get_release_date(origin_match)) + + origin_match.extra_data["release_date"] = "2021-05-05" + self.assertEqual(date(2021, 5, 5), origin.get_release_date(origin_match)) + + origin_match.extra_data["release_date"] = "May 5, 2021" + self.assertIsNone(origin.get_release_date(origin_match)) + def test_scanpipe_pipes_origin_get_versionless_purl(self): purl = "pkg:pypi/six@1.17.0?file_name=six-1.17.0.tar.gz#six.py" self.assertEqual("pkg:pypi/six", origin.get_versionless_purl(purl)) @@ -184,6 +298,43 @@ def test_scanpipe_pipes_origin_origin_versionless_purls(self): expected = {"pkg:github/benjaminp/six", "pkg:pypi/six", "pkg:conda/six"} self.assertEqual(expected, make_scanoss_six_origin().versionless_purls) + def test_scanpipe_pipes_origin_origin_package_names(self): + self.assertEqual({"six"}, make_scanoss_six_origin().package_names) + typing_origin = make_origin("pkg:pypi/Typing-Extensions@4.9.0") + self.assertEqual({"typing_extensions"}, typing_origin.package_names) + + def test_scanpipe_pipes_origin_origin_copy_names(self): + pip_origin = make_origin("pkg:pypi/pip@24.0") + self.assertEqual(set(), pip_origin.copy_names) + + pip_origin.merge( + make_origin("pkg:pypi/pip@24.0", origin_path="pip-24.0/_vendor/six.py") + ) + self.assertEqual({"six"}, pip_origin.copy_names) + + def test_scanpipe_pipes_origin_origin_is_release_version(self): + commit_origin = make_origin("pkg:github/benhoyt/inih@498f34b", tool="scanoss") + self.assertFalse(commit_origin.is_release_version) + self.assertTrue(make_origin("pkg:github/benhoyt/inih@r62").is_release_version) + versionless_origin = make_origin("pkg:github/benhoyt/inih", tool="scanoss") + self.assertFalse(versionless_origin.is_release_version) + + def test_scanpipe_pipes_origin_origin_similarity(self): + six_origin = make_origin("pkg:pypi/six@1.17.0") + self.assertIsNone(six_origin.similarity) + + six_origin.merge(make_origin("pkg:pypi/six@1.17.0", similarity=94)) + six_origin.merge(make_origin("pkg:pypi/six@1.17.0", similarity=95)) + self.assertEqual(95, six_origin.similarity) + + def test_scanpipe_pipes_origin_origin_release_date(self): + six_origin = make_origin("pkg:pypi/six@1.16.0") + self.assertIsNone(six_origin.release_date) + + six_origin.merge(make_origin("pkg:pypi/six@1.16.0", release_date="2021-06-01")) + six_origin.merge(make_origin("pkg:pypi/six@1.16.0", release_date="2021-05-05")) + self.assertEqual(date(2021, 5, 5), six_origin.release_date) + def test_scanpipe_pipes_origin_origin_is_same_package(self): scanoss_origin = make_scanoss_six_origin() self.assertTrue( @@ -238,6 +389,19 @@ def test_scanpipe_pipes_origin_group_origin_matches(self): ) self.assertEqual({"matchcode", "scanoss"}, origins[1].tools) + # The "v" prefix of a SCANOSS Git tag does not split the package. + origin_matches = [ + make_unsaved_origin_match("pkg:npm/escape-html@1.0.3"), + make_unsaved_origin_match( + "pkg:github/component/escape-html@v1.0.3", + tool="scanoss", + purls=["pkg:npm/escape-html"], + ), + ] + origins = origin.group_origin_matches(origin_matches) + self.assertEqual(1, len(origins)) + self.assertEqual("pkg:npm/escape-html@1.0.3", origins[0].purl) + def test_scanpipe_pipes_origin_iter_resource_origins(self): six_resource = make_resource_file(self.project1, path="six.py") ini_resource = make_resource_file(self.project1, path="ini.c") @@ -306,24 +470,337 @@ def test_scanpipe_pipes_origin_is_same_package_versions(self): origin.is_same_package_versions([six_16_origin, urllib3_origin]) ) - def test_scanpipe_pipes_origin_get_rule(self): - six_origin = make_origin("pkg:pypi/six@1.17.0") + def test_scanpipe_pipes_origin_is_carrier_origin(self): + six_origin = make_origin("pkg:pypi/six@1.16.0", origin_path="six-1.16.0/six.py") + pip_origin = make_origin( + "pkg:pypi/pip@24.0", origin_path="pip-24.0/src/pip/_vendor/six.py" + ) + urllib3_origin = make_origin( + "pkg:pypi/urllib3@1.26.20", + origin_path="urllib3-1.26.20/src/urllib3/packages/six.py", + ) + + self.assertTrue(origin.is_carrier_origin(pip_origin, [six_origin])) + self.assertFalse(origin.is_carrier_origin(six_origin, [pip_origin])) + self.assertFalse(origin.is_carrier_origin(urllib3_origin, [pip_origin])) + # Without origin path, the content of a package is unknown. + self.assertFalse( + origin.is_carrier_origin(make_origin("pkg:pypi/pip@24.0"), [six_origin]) + ) + + # A directory that is not under a vendor directory. + foo_origin = make_origin( + "pkg:npm/foo@1.0.0", origin_path="foo-1.0/src/utils/index.js" + ) + utils_origin = make_origin("pkg:npm/utils@0.3.1") + self.assertFalse(origin.is_carrier_origin(foo_origin, [utils_origin])) + + jquery_origin = make_origin("pkg:npm/jquery@3.7.1") + bootstrap_origin = make_origin( + "pkg:npm/bootstrap@5.3.3", + origin_path="bootstrap-5.3.3/site/static/vendor/jquery.min.js", + ) + self.assertTrue(origin.is_carrier_origin(bootstrap_origin, [jquery_origin])) + + def test_scanpipe_pipes_origin_get_non_carrier_origins(self): + six_origin = make_origin("pkg:pypi/six@1.16.0", origin_path="six-1.16.0/six.py") + pip_origin = make_origin( + "pkg:pypi/pip@24.0", origin_path="pip-24.0/src/pip/_vendor/six.py" + ) + virtualenv_origin = make_origin( + "pkg:pypi/virtualenv@20.0.0", + origin_path="virtualenv-20.0.0/src/virtualenv/six.py", + ) + # SCANOSS reports the file name only. + anime365_origin = make_origin( + "pkg:github/piotrmasek/anime365@0.1-Ryucon", + tool="scanoss", + origin_path="six.py", + ) + lodash_origin = make_origin( + "pkg:npm/lodash@4.17.21", origin_path="package/debounce.js" + ) + + non_carrier_origins = origin.get_non_carrier_origins([pip_origin, six_origin]) + self.assertEqual([six_origin], non_carrier_origins) + non_carrier_origins = origin.get_non_carrier_origins( + [pip_origin, virtualenv_origin, six_origin] + ) + self.assertEqual([six_origin], non_carrier_origins) + non_carrier_origins = origin.get_non_carrier_origins( + [anime365_origin, six_origin] + ) + self.assertEqual([six_origin], non_carrier_origins) + + # No carrier. + non_carrier_origins = origin.get_non_carrier_origins( + [lodash_origin, six_origin] + ) + self.assertEqual([lodash_origin, six_origin], non_carrier_origins) + # Several origins that do not carry another. + non_carrier_origins = origin.get_non_carrier_origins( + [pip_origin, six_origin, lodash_origin] + ) + self.assertEqual([six_origin, lodash_origin], non_carrier_origins) + + # Regression test: the versions of six carried each other, and the copier + # was proposed. + six_17_origin = make_origin( + "pkg:pypi/six@1.17.0", origin_path="six-1.17.0/six.py" + ) + foo_origin = make_origin("pkg:pypi/foo@1.0", origin_path="foo/compat.py") + self.assertFalse(origin.is_carrier_origin(six_origin, [six_17_origin])) + non_carrier_origins = origin.get_non_carrier_origins( + [six_origin, six_17_origin, foo_origin] + ) + self.assertEqual([six_origin, six_17_origin, foo_origin], non_carrier_origins) + + def test_scanpipe_pipes_origin_get_earliest_release_origin(self): + six_origin = make_origin("pkg:pypi/six@1.16.0", release_date="2021-05-05") + pip_origin = make_origin("pkg:pypi/pip@24.0", release_date="2024-02-03") + urllib3_origin = make_origin("pkg:pypi/urllib3@1.26.20") + + earliest_origin = origin.get_earliest_release_origin([pip_origin, six_origin]) + self.assertEqual(six_origin, earliest_origin) + + # A copy is not dated without the release date of all the candidates. + earliest_origin = origin.get_earliest_release_origin( + [pip_origin, six_origin, urllib3_origin] + ) + self.assertIsNone(earliest_origin) + + same_day_origin = make_origin("pkg:pypi/ms@2.1.3", release_date="2021-05-05") + earliest_origin = origin.get_earliest_release_origin( + [pip_origin, six_origin, same_day_origin] + ) + self.assertIsNone(earliest_origin) + + def test_scanpipe_pipes_origin_get_most_similar_origin(self): + lodash_origin = make_origin("pkg:npm/lodash@4.17.21", similarity=75) + underscore_origin = make_origin("pkg:npm/underscore@1.13.7", similarity=40) + debounce_origin = make_origin("pkg:npm/debounce@2.2.0", similarity=60) + + most_similar_origin = origin.get_most_similar_origin( + [underscore_origin, lodash_origin] + ) + self.assertEqual(lodash_origin, most_similar_origin) + + # The second highest similarity is within the margin. + most_similar_origin = origin.get_most_similar_origin( + [underscore_origin, lodash_origin, debounce_origin] + ) + self.assertIsNone(most_similar_origin) + + exact_six_origin = make_origin("pkg:pypi/six@1.16.0", similarity=100) + exact_pip_origin = make_origin("pkg:pypi/pip@24.0", similarity=100) + most_similar_origin = origin.get_most_similar_origin( + [exact_six_origin, exact_pip_origin] + ) + self.assertIsNone(most_similar_origin) + + # A directory match has no similarity. + directory_origin = make_origin("pkg:npm/ms@2.1.3", match_type="directory") + most_similar_origin = origin.get_most_similar_origin( + [lodash_origin, directory_origin] + ) + self.assertIsNone(most_similar_origin) + + # A snippet in a large file is rounded to a known 0 similarity. + rounded_origin = make_origin("pkg:npm/ms@2.1.3", similarity=0) + most_similar_origin = origin.get_most_similar_origin( + [rounded_origin, underscore_origin] + ) + self.assertEqual(underscore_origin, most_similar_origin) + + def test_scanpipe_pipes_origin_get_preferred_version_origin(self): + r62_origin = make_origin("pkg:github/benhoyt/inih@r62", similarity=90) + commit_origin = make_origin( + "pkg:github/benhoyt/inih@498f34b", tool="scanoss", similarity=94 + ) + r61_origin = make_origin("pkg:github/benhoyt/inih@r61", similarity=60) + other_commit_origin = make_origin( + "pkg:github/benhoyt/inih@577ae2d", tool="scanoss", similarity=60 + ) + + # The release is preferred over a commit with a similarity within the margin. + preferred_origin = origin.get_preferred_version_origin( + [r62_origin, commit_origin] + ) + self.assertEqual(r62_origin, preferred_origin) + + # An exact copy of a commit is preferred over a short release snippet. + exact_commit_origin = make_origin( + "pkg:github/benhoyt/inih@498f34b", tool="scanoss", similarity=100 + ) + snippet_release_origin = make_origin( + "pkg:github/benhoyt/inih@r1", match_type="snippet", similarity=10 + ) + preferred_origin = origin.get_preferred_version_origin( + [snippet_release_origin, exact_commit_origin] + ) + self.assertEqual(exact_commit_origin, preferred_origin) + + # Between releases, the clearly most similar. + preferred_origin = origin.get_preferred_version_origin( + [r61_origin, r62_origin, commit_origin] + ) + self.assertEqual(r62_origin, preferred_origin) + + # Between commits only, the clearly most similar. + preferred_origin = origin.get_preferred_version_origin( + [other_commit_origin, commit_origin] + ) + self.assertEqual(commit_origin, preferred_origin) + + # No preference within the margin, or on unknown similarities. + r60_origin = make_origin("pkg:github/benhoyt/inih@r60", similarity=80) + self.assertIsNone(origin.get_preferred_version_origin([r60_origin, r62_origin])) + six_17_origin = make_origin("pkg:pypi/six@1.17.0") + six_16_origin = make_origin("pkg:pypi/six@1.16.0") + self.assertIsNone( + origin.get_preferred_version_origin([six_17_origin, six_16_origin]) + ) + + # The same file in several releases: the earliest release. + dated_six_17_origin = make_origin( + "pkg:pypi/six@1.17.0", similarity=100, release_date="2024-12-04" + ) + dated_six_16_origin = make_origin( + "pkg:pypi/six@1.16.0", similarity=100, release_date="2021-05-05" + ) + preferred_origin = origin.get_preferred_version_origin( + [dated_six_17_origin, dated_six_16_origin] + ) + self.assertEqual(dated_six_16_origin, preferred_origin) + + # Regression test: a versionless candidate raised a TypeError. + versionless_origin = make_origin("pkg:pypi/six", tool="scanoss") + preferred_origin = origin.get_preferred_version_origin( + [versionless_origin, six_16_origin] + ) + self.assertEqual(six_16_origin, preferred_origin) + versionless_inih_origin = make_origin("pkg:github/benhoyt/inih", tool="scanoss") + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [versionless_inih_origin, commit_origin] + ) + self.assertEqual((origin.CONFLICTING_VERSIONS_RULE, None), rule_and_candidate) + + def test_scanpipe_pipes_origin_get_conflict_rule_and_candidate(self): + r62_origin = make_origin("pkg:github/benhoyt/inih@r62", similarity=95) + commit_origin = make_origin( + "pkg:github/benhoyt/inih@498f34b", tool="scanoss", similarity=94 + ) + six_17_origin = make_origin("pkg:pypi/six@1.17.0") six_16_origin = make_origin("pkg:pypi/six@1.16.0") urllib3_origin = make_origin("pkg:pypi/urllib3@1.26.20") + + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [six_16_origin, urllib3_origin] + ) + self.assertEqual((origin.CONFLICTING_CANDIDATES_RULE, None), rule_and_candidate) + + pip_origin = make_origin( + "pkg:pypi/pip@24.0", origin_path="pip-24.0/src/pip/_vendor/six.py" + ) + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [pip_origin, six_16_origin] + ) + self.assertEqual((origin.CONTAINMENT_RULE, six_16_origin), rule_and_candidate) + + # The carrier is removed before comparing the versions of six, even without + # its release date. + dated_six_17_origin = make_origin( + "pkg:pypi/six@1.17.0", similarity=100, release_date="2024-12-04" + ) + dated_six_16_origin = make_origin( + "pkg:pypi/six@1.16.0", similarity=100, release_date="2021-05-05" + ) + carrier_pip_origin = make_origin( + "pkg:pypi/pip@24.0", + similarity=100, + origin_path="pip-24.0/src/pip/_vendor/six.py", + ) + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [dated_six_17_origin, carrier_pip_origin, dated_six_16_origin] + ) + expected = (origin.PREFERRED_VERSION_RULE, dated_six_16_origin) + self.assertEqual(expected, rule_and_candidate) + + # The release date does not decide between different packages. + dated_rich_origin = make_origin( + "pkg:pypi/rich@13.7.0", similarity=100, release_date="2023-11-15" + ) + dated_copy_origin = make_origin( + "pkg:github/dreamsavior/yojet-server@v1.0.1", + tool="scanoss", + similarity=100, + release_date="2022-08-01", + ) + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [dated_rich_origin, dated_copy_origin] + ) + self.assertEqual((origin.CONFLICTING_CANDIDATES_RULE, None), rule_and_candidate) + + lodash_origin = make_origin("pkg:npm/lodash@4.17.21", similarity=75) + underscore_origin = make_origin("pkg:npm/underscore@1.13.7", similarity=40) + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [underscore_origin, lodash_origin] + ) + expected = (origin.HIGHEST_SIMILARITY_RULE, lodash_origin) + self.assertEqual(expected, rule_and_candidate) + + # An exact copy is preferred over an older package matched on a snippet. + exact_pip_origin = make_origin( + "pkg:pypi/pip@24.0", similarity=100, release_date="2024-02-03" + ) + older_snippet_origin = make_origin( + "pkg:pypi/six@1.16.0", + match_type="snippet", + similarity=10, + release_date="2021-05-05", + ) + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [older_snippet_origin, exact_pip_origin] + ) + expected = (origin.HIGHEST_SIMILARITY_RULE, exact_pip_origin) + self.assertEqual(expected, rule_and_candidate) + + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [r62_origin, commit_origin] + ) + self.assertEqual( + (origin.PREFERRED_VERSION_RULE, r62_origin), rule_and_candidate + ) + + rule_and_candidate = origin.get_conflict_rule_and_candidate( + [six_17_origin, six_16_origin] + ) + self.assertEqual((origin.CONFLICTING_VERSIONS_RULE, None), rule_and_candidate) + + def test_scanpipe_pipes_origin_get_single_origin_rule(self): + six_origin = make_origin("pkg:pypi/six@1.17.0") snippet_origin = make_origin("pkg:npm/lodash@4.17.21", match_type="snippet") - rule = origin.get_rule([six_origin, urllib3_origin]) - self.assertEqual(origin.CONFLICTING_CANDIDATES_RULE, rule) - rule = origin.get_rule([six_origin, six_16_origin]) - self.assertEqual(origin.CONFLICTING_VERSIONS_RULE, rule) - rule = origin.get_rule([snippet_origin]) + rule = origin.get_single_origin_rule(snippet_origin) self.assertEqual(origin.NO_EXACT_MATCH_RULE, rule) - self.assertEqual(origin.SINGLE_TOOL_RULE, origin.get_rule([six_origin])) + self.assertEqual( + origin.SINGLE_TOOL_RULE, origin.get_single_origin_rule(six_origin) + ) six_origin.merge(make_scanoss_six_origin()) - rule = origin.get_rule([six_origin]) + rule = origin.get_single_origin_rule(six_origin) self.assertEqual(origin.CONFIRMED_BY_TWO_TOOLS_RULE, rule) + def test_scanpipe_pipes_origin_get_rule_and_candidate(self): + six_origin = make_origin("pkg:pypi/six@1.17.0") + urllib3_origin = make_origin("pkg:pypi/urllib3@1.26.20") + + rule_and_candidate = origin.get_rule_and_candidate([six_origin]) + self.assertEqual((origin.SINGLE_TOOL_RULE, six_origin), rule_and_candidate) + + rule_and_candidate = origin.get_rule_and_candidate([six_origin, urllib3_origin]) + self.assertEqual((origin.CONFLICTING_CANDIDATES_RULE, None), rule_and_candidate) + def test_scanpipe_pipes_origin_make_origin_determination(self): resource = make_resource_file(self.project1, path="six.py") six_origin = make_origin("pkg:pypi/six@1.17.0") @@ -344,6 +821,18 @@ def test_scanpipe_pipes_origin_make_origin_determination(self): self.assertEqual("needs-review", determination.status) self.assertEqual(origin.CONFLICTING_CANDIDATES_RULE, determination.rule) + # The preferred version is proposed for review. + r62_origin = make_origin("pkg:github/benhoyt/inih@r62", similarity=95) + commit_origin = make_origin( + "pkg:github/benhoyt/inih@498f34b", tool="scanoss", similarity=94 + ) + determination = origin.make_origin_determination( + self.project1, resource, [r62_origin, commit_origin] + ) + self.assertEqual("pkg:github/benhoyt/inih@r62", determination.purl) + self.assertEqual("needs-review", determination.status) + self.assertEqual(origin.PREFERRED_VERSION_RULE, determination.rule) + six_origin.merge(make_scanoss_six_origin()) determination = origin.make_origin_determination( self.project1, resource, [six_origin] @@ -459,13 +948,20 @@ def test_scanpipe_pipes_origin_assign_origin_package_resources(self): expected = {origin.ORIGIN_RESOURCE_PATHS_KEY: ["six.py"]} self.assertEqual(expected, scan_package.extra_data) + # Regression test: the paths of a second origin PURL replaced the first. + test_six = make_resource_file(self.project1, path="test_six.py") + origin.assign_origin_package_resources(scan_package, resources) + expected = {origin.ORIGIN_RESOURCE_PATHS_KEY: ["six.py", "test_six.py"]} + self.assertEqual(expected, scan_package.extra_data) + self.assertIn(test_six, scan_package.codebase_resources.all()) + origin_package = make_package( self.project1, "pkg:npm/ms@2.1.3", datasource_ids=[origin.ORIGIN_DATASOURCE_ID], ) origin.assign_origin_package_resources(origin_package, resources) - self.assertEqual(2, origin_package.codebase_resources.count()) + self.assertEqual(3, origin_package.codebase_resources.count()) self.assertEqual({}, origin_package.extra_data) def test_scanpipe_pipes_origin_unassign_scan_package_resources(self): @@ -573,6 +1069,36 @@ def test_scanpipe_pipes_origin_get_origin_packages(self): origin.get_origin_packages(self.project1), [origin_package] ) + def test_scanpipe_pipes_origin_get_scan_package(self): + purl = "pkg:maven/org.slf4j/slf4j-nop@2.0.16" + self.assertIsNone(origin.get_scan_package(self.project1, purl)) + + make_package( + self.project1, + "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + datasource_ids=[origin.ORIGIN_DATASOURCE_ID], + ) + make_package(self.project1, "pkg:maven/org.slf4j/slf4j-nop@2.0.15") + self.assertIsNone(origin.get_scan_package(self.project1, purl)) + + sources_package = make_package( + self.project1, "pkg:maven/org.slf4j/slf4j-nop@2.0.16?classifier=sources" + ) + self.assertEqual(sources_package, origin.get_scan_package(self.project1, purl)) + + jar_package = make_package( + self.project1, "pkg:maven/org.slf4j/slf4j-nop@2.0.16?type=jar" + ) + no_qualifiers_package = make_package( + self.project1, "pkg:maven/org.slf4j/slf4j-nop@2.0.16" + ) + self.assertEqual( + no_qualifiers_package, origin.get_scan_package(self.project1, purl) + ) + + jar_purl = "pkg:maven/org.slf4j/slf4j-nop@2.0.16?type=jar" + self.assertEqual(jar_package, origin.get_scan_package(self.project1, jar_purl)) + def test_scanpipe_pipes_origin_get_or_create_origin_package(self): scan_package = make_package(self.project1, "pkg:npm/ms@2.1.3") resources = self.project1.codebaseresources.all() @@ -581,6 +1107,15 @@ def test_scanpipe_pipes_origin_get_or_create_origin_package(self): ) self.assertEqual(scan_package, package) + # Regression test: the scan package with qualifiers was not found. + sources_package = make_package( + self.project1, "pkg:maven/org.slf4j/slf4j-nop@2.0.16?classifier=sources" + ) + package = origin.get_or_create_origin_package( + self.project1, "pkg:maven/org.slf4j/slf4j-nop@2.0.16", resources + ) + self.assertEqual(sources_package, package) + resource = make_resource_file(self.project1, path="six.py") self.make_origin_match( resource, @@ -726,12 +1261,13 @@ def test_scanpipe_pipes_origin_determine_origins_code_genetics_data_set(self): origin.CONFLICTING_CANDIDATES_RULE, ), ], + # The r62 release is preferred over the 498f34b commit. "modified-copy": [ ( "src/inih/ini.c", - "", + "pkg:github/benhoyt/inih@r62", "needs-review", - origin.CONFLICTING_VERSIONS_RULE, + origin.PREFERRED_VERSION_RULE, ), ( "src/inih/ini.h", @@ -757,6 +1293,33 @@ def test_scanpipe_pipes_origin_determine_origins_code_genetics_data_set(self): ), ], "no-origin": [], + # The origin six and pip, which vendors six, are both reported. + "shared-copy": [ + ( + "third_party/six.py", + "pkg:pypi/six@1.16.0", + "needs-review", + origin.CONTAINMENT_RULE, + ), + ], + "embedded-archive": [ + ( + "jars/slf4j-nop-2.0.16.jar", + "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + "auto-confirmed", + origin.CONFIRMED_BY_TWO_TOOLS_RULE, + ), + ], + "single-tool-copy": [ + ( + "static/js/escape-html.js", + "pkg:github/component/escape-html@v1.0.3", + "needs-review", + origin.SINGLE_TOOL_RULE, + ), + ], + # The license header snippet matches are too short to count. + "license-header": [], } expected_packages_by_case = { @@ -777,6 +1340,13 @@ def test_scanpipe_pipes_origin_determine_origins_code_genetics_data_set(self): ], ) ], + "embedded-archive": [ + ( + "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + "mit", + ["jars/slf4j-nop-2.0.16.jar"], + ) + ], } for case_name, expected in expected_by_case.items(): @@ -833,6 +1403,19 @@ def test_scanpipe_pipes_origin_determine_origins_data_set_single_tool(self): ("vendor/ms", "pkg:npm/ms@2.1.3", "no-exact-match"), ], "no-origin": [], + "shared-copy": [ + ("third_party/six.py", "pkg:pypi/six@1.16.0", "containment"), + ], + "embedded-archive": [ + ( + "jars/slf4j-nop-2.0.16.jar", + "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + "single-tool", + ), + ], + # Not in the MatchCode.io index. + "single-tool-copy": [], + "license-header": [], }, scanoss.TOOL_NAME: { "exact-copy": [ @@ -862,6 +1445,28 @@ def test_scanpipe_pipes_origin_determine_origins_data_set_single_tool(self): ("vendor/ms/index.js", "pkg:github/vercel/ms@2.1.3", "single-tool"), ], "no-origin": [], + "shared-copy": [ + ( + "third_party/six.py", + "pkg:github/benjaminp/six@1.16.0", + "single-tool", + ), + ], + "embedded-archive": [ + ( + "jars/slf4j-nop-2.0.16.jar", + "pkg:maven/org.slf4j/slf4j-nop@2.0.16", + "single-tool", + ), + ], + "single-tool-copy": [ + ( + "static/js/escape-html.js", + "pkg:github/component/escape-html@v1.0.3", + "single-tool", + ), + ], + "license-header": [], }, } diff --git a/scanpipe/tests/pipes/test_pipes.py b/scanpipe/tests/pipes/test_pipes.py index 6a6987ae2b..6c8cbf9965 100644 --- a/scanpipe/tests/pipes/test_pipes.py +++ b/scanpipe/tests/pipes/test_pipes.py @@ -122,6 +122,66 @@ def make_origin_matches(resource, purls, tool="test"): six_resource.origin_matches.filter(pk=other_tool_match.pk).exists() ) + def test_scanpipe_pipes_create_origin_matches_short_snippet_matches(self): + project1 = make_project() + make_resource_file(project1, "header.py") + debounce_resource = make_resource_file(project1, "debounce.js") + line_ranges_by_purl = { + "pkg:npm/lodash@4.17.21": [{"lines": [26, 140]}], + "pkg:pypi/six@1.16.0": [{"lines": [3, 19]}], + } + results_by_path = { + "header.py": ["pkg:pypi/six@1.16.0"], + "debounce.js": ["pkg:npm/lodash@4.17.21", "pkg:pypi/six@1.16.0"], + } + + def make_origin_matches(resource, purls): + for purl in purls: + yield OriginMatch( + project=project1, + resource=resource, + purl=purl, + match_type=OriginMatch.MatchType.SNIPPET, + line_ranges=line_ranges_by_purl[purl], + tool="test", + ) + + origin_match_count, _ = pipes.create_origin_matches( + project1, "test", results_by_path, make_origin_matches + ) + self.assertEqual(1, origin_match_count) + origin_match = project1.originmatchs.get() + self.assertEqual(debounce_resource, origin_match.resource) + self.assertEqual("pkg:npm/lodash@4.17.21", origin_match.purl) + + def test_scanpipe_pipes_get_matched_line_count(self): + origin_match = OriginMatch(line_ranges=[]) + self.assertEqual(0, pipes.get_matched_line_count(origin_match)) + + origin_match.line_ranges = [{"lines": [3, 19]}] + self.assertEqual(17, pipes.get_matched_line_count(origin_match)) + + # Overlapping ranges count each line once. + origin_match.line_ranges = [ + {"lines": [1, 36], "origin_lines": [1, 36]}, + {"lines": [30, 40], "origin_lines": [30, 40]}, + ] + self.assertEqual(40, pipes.get_matched_line_count(origin_match)) + + def test_scanpipe_pipes_is_short_snippet_match(self): + origin_match = OriginMatch(match_type="snippet", line_ranges=[]) + self.assertTrue(pipes.is_short_snippet_match(origin_match)) + + origin_match.line_ranges = [{"lines": [1, 19]}] + self.assertTrue(pipes.is_short_snippet_match(origin_match)) + + origin_match.line_ranges = [{"lines": [1, 20]}] + self.assertFalse(pipes.is_short_snippet_match(origin_match)) + + # Only the snippet matches have a minimum length. + file_match = OriginMatch(match_type="file", line_ranges=[]) + self.assertFalse(pipes.is_short_snippet_match(file_match)) + def test_scanpipe_pipes_make_origin_match(self): project1 = make_project() resource = make_resource_file(project1, "six.py") diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index 5a68afa53d..a393b47595 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -243,6 +243,26 @@ def test_scanpipe_pipes_purldb_check_project_run_statuses( self.assertEqual(expected_data, mock_request_post_call_kwargs["data"]) self.project1.runs.all().delete() + @mock.patch("scanpipe.pipes.purldb.request_post") + def test_scanpipe_pipes_purldb_collect_response_results(self, mock_request_post): + first_page = {"count": 3, "results": [1], "next": "http://purldb/?page=2"} + second_page = {"count": 3, "results": [2], "next": "http://purldb/?page=3"} + last_page = {"count": 3, "results": [3], "next": None} + mock_request_post.side_effect = [second_page, last_page] + + results = purldb.collect_response_results(first_page, data={}) + self.assertEqual([1, 2, 3], results) + self.assertEqual(2, mock_request_post.call_count) + + # Regression test: a failed next page request looped forever. + mock_request_post.reset_mock() + mock_request_post.side_effect = [second_page, None] + results = purldb.collect_response_results(first_page, data={}) + self.assertEqual([1, 2], results) + self.assertEqual(2, mock_request_post.call_count) + + self.assertEqual([], purldb.collect_response_results(None, data={})) + def test_scanpipe_pipes_purldb_create_project_name(self): download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" scannable_uri_uuid = "52b2930d-6e85-4b3e-ba3e-17dd9a618650" diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index df541e6f6f..f335fd3606 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1675,6 +1675,7 @@ def test_scanpipe_fetch_scores_pipeline_integration(self, mock_is_available): self.assertEqual("v5.2.1", scorecard_entry.scoring_tool_version) self.assertTrue(scorecard_entry.score) + @mock.patch("scanpipe.pipes.purldb.is_configured") @mock.patch("scanpipe.pipes.scanoss.scan_codebase") @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") @mock.patch("scanpipe.pipes.scanoss.is_configured") @@ -1687,6 +1688,7 @@ def test_scanpipe_find_origins_pipeline_integration( mock_scanoss_is_configured, mock_get_uvx_location, mock_scan_codebase, + mock_purldb_is_configured, ): pipeline_name = "find_origins" project1 = make_project() @@ -1713,11 +1715,13 @@ def write_scanoss_results(codebase_path, output_location): mock_scanoss_is_configured.return_value = True mock_get_uvx_location.return_value = "/usr/bin/uvx" mock_scan_codebase.side_effect = write_scanoss_results + mock_purldb_is_configured.return_value = False run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) + self.assertIn("PurlDB is not configured", run.log) origin_matches = project1.originmatchs.order_by("tool", "resource__path") expected = [