diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index fc1e490eeff..fd95f0b26e9 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -171,6 +171,7 @@ pypi.PypiEggHandler, # pypi.PypiSdistArchiveHandler, pypi.PypiWheelHandler, + pypi.PipWheelCacheOriginJsonHandler, pypi.PyprojectTomlHandler, pypi.PoetryPyprojectTomlHandler, pypi.PoetryLockHandler, diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index dcfd237590e..c417e7c9cba 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -22,6 +22,7 @@ from fnmatch import fnmatchcase from pathlib import Path from typing import NamedTuple +from urllib.parse import urlsplit import dparse2 # NOTE: we always want to use the external library rather than the built-in for now @@ -37,6 +38,11 @@ from packvers import markers from packvers.requirements import Requirement from packvers.utils import canonicalize_name +from packvers.utils import InvalidSdistFilename +from packvers.utils import InvalidWheelFilename +from packvers.utils import parse_sdist_filename +from packvers.utils import parse_wheel_filename +from packvers.version import InvalidVersion from packagedcode import models from packagedcode.utils import build_description @@ -1445,6 +1451,114 @@ def parse(cls, location, package_only=False): yield models.PackageData.from_data(package_data, package_only) +def parse_distribution_filename(filename): + """ + Return a (name, version) tuple parsed from a wheel or sdist file name + such as ``pefile-2022.5.30-py3-none-any.whl`` or + ``pefile-2022.5.30.tar.gz``. Return None if the file name cannot be + parsed as a distribution file name. + """ + if not filename: + return None + try: + if filename.endswith('.whl'): + name, version, _build, _tags = parse_wheel_filename(filename) + else: + name, version = parse_sdist_filename(filename) + except (InvalidWheelFilename, InvalidSdistFilename, InvalidVersion): + return None + return str(name), str(version) + + +class PipWheelCacheOriginJsonHandler(models.DatafileHandler): + datasource_id = 'pypi_pip_cache_origin_json' + path_patterns = ('*origin.json',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'pip wheel cache origin.json' + documentation_url = 'https://pip.pypa.io/en/stable/topics/caching/' + + @classmethod + def parse(cls, location, package_only=False): + try: + with open(location) as f: + data = json.load(f) + except (OSError, ValueError): + # unreadable or invalid JSON: not a pip cache origin file + return + + if not isinstance(data, dict): + return + + url = data.get('url') + if not isinstance(url, str) or not url: + url = None + + archive_info = data.get('archive_info') or {} + if not isinstance(archive_info, dict): + archive_info = {} + + sha256 = None + hashes = archive_info.get('hashes') or {} + if isinstance(hashes, dict): + sha256 = hashes.get('sha256') + if not isinstance(sha256, str): + sha256 = None + if not sha256: + # the single hash is recorded as e.g. "sha256=" + hash_value = archive_info.get('hash') or '' + if isinstance(hash_value, str): + for prefix in ('sha256:', 'sha256='): + if hash_value.startswith(prefix): + sha256 = hash_value[len(prefix):] + break + + name, version = cls.get_name_version(location, url) + if not name: + return + + urls = get_pypi_urls(name, version) + if url: + # prefer the exact URL recorded in the cache origin file + urls['repository_download_url'] = url + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language=cls.default_primary_language, + name=name, + version=version, + download_url=url, + sha256=sha256, + **urls, + ) + yield models.PackageData.from_data(package_data, package_only) + + @classmethod + def get_name_version(cls, location, url): + """ + Return a (name, version) tuple for the package described by a pip + cache ``origin.json`` file, parsed from the download ``url`` file name + or, if there is no usable URL, from the name of a sibling wheel file + in the same cache directory. Return (None, None) if no name and + version can be determined. + """ + candidates = [] + if url: + candidates.append(posixpath.basename(urlsplit(url).path)) + # a pip wheel cache directory holds the built wheel next to origin.json + location_path = Path(location) + if location_path.parent.is_dir(): + candidates.extend( + sorted(p.name for p in location_path.parent.glob('*.whl')) + ) + for filename in candidates: + name_version = parse_distribution_filename(filename) + if name_version: + return name_version + return None, None + + class PythonSetupPyHandler(BaseExtractedPythonLayout): datasource_id = 'pypi_setup_py' path_patterns = ('*setup.py',) diff --git a/tests/packagedcode/data/plugin/plugins_list_linux.txt b/tests/packagedcode/data/plugin/plugins_list_linux.txt old mode 100755 new mode 100644 index 34078bc580b..b67aee07e48 --- a/tests/packagedcode/data/plugin/plugins_list_linux.txt +++ b/tests/packagedcode/data/plugin/plugins_list_linux.txt @@ -748,6 +748,13 @@ Package type: pypi description: Python poetry pyproject.toml path_patterns: '*pip-inspect.deplock' -------------------------------------------- +Package type: pypi + datasource_id: pypi_pip_cache_origin_json + documentation URL: https://pip.pypa.io/en/stable/topics/caching/ + primary language: Python + description: pip wheel cache origin.json + path_patterns: '*origin.json' +-------------------------------------------- Package type: pypi datasource_id: pypi_poetry_lock documentation URL: https://python-poetry.org/docs/basic-usage/#installing-with-poetrylock diff --git a/tests/packagedcode/data/pypi/pip-cache-origin-json/invalid/origin.json b/tests/packagedcode/data/pypi/pip-cache-origin-json/invalid/origin.json new file mode 100644 index 00000000000..512de215596 --- /dev/null +++ b/tests/packagedcode/data/pypi/pip-cache-origin-json/invalid/origin.json @@ -0,0 +1 @@ +this is not JSON at all diff --git a/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-hash/origin.json b/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-hash/origin.json new file mode 100644 index 00000000000..7e0d77c1cfd --- /dev/null +++ b/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-hash/origin.json @@ -0,0 +1 @@ +{"archive_info":{},"url":"https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz"} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-url/origin.json b/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-url/origin.json new file mode 100644 index 00000000000..8f07b336c15 --- /dev/null +++ b/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-url/origin.json @@ -0,0 +1 @@ +{"archive_info":{"hash":"sha256=a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b","hashes":{"sha256":"a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b"}}} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-url/pefile-2022.5.30-py3-none-any.whl b/tests/packagedcode/data/pypi/pip-cache-origin-json/missing-url/pefile-2022.5.30-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile-origin.json-expected.json b/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile-origin.json-expected.json new file mode 100644 index 00000000000..bf320c671dd --- /dev/null +++ b/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile-origin.json-expected.json @@ -0,0 +1,46 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "pefile", + "version": "2022.5.30", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": "a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/pefile", + "repository_download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz", + "api_data_url": "https://pypi.org/pypi/pefile/2022.5.30/json", + "datasource_id": "pypi_pip_cache_origin_json", + "purl": "pkg:pypi/pefile@2022.5.30" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile-package-expected.json b/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile-package-expected.json new file mode 100644 index 00000000000..b95777c2f9f --- /dev/null +++ b/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile-package-expected.json @@ -0,0 +1,109 @@ +{ + "packages": [ + { + "type": "pypi", + "namespace": null, + "name": "pefile", + "version": "2022.5.30", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": "a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "repository_homepage_url": "https://pypi.org/project/pefile", + "repository_download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz", + "api_data_url": "https://pypi.org/pypi/pefile/2022.5.30/json", + "package_uid": "pkg:pypi/pefile@2022.5.30?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "origin.json" + ], + "datasource_ids": [ + "pypi_pip_cache_origin_json" + ], + "purl": "pkg:pypi/pefile@2022.5.30" + } + ], + "dependencies": [], + "files": [ + { + "path": "origin.json", + "type": "file", + "package_data": [ + { + "type": "pypi", + "namespace": null, + "name": "pefile", + "version": "2022.5.30", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": "a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/pefile", + "repository_download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz", + "api_data_url": "https://pypi.org/pypi/pefile/2022.5.30/json", + "datasource_id": "pypi_pip_cache_origin_json", + "purl": "pkg:pypi/pefile@2022.5.30" + } + ], + "for_packages": [ + "pkg:pypi/pefile@2022.5.30?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile/origin.json b/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile/origin.json new file mode 100644 index 00000000000..fffdf5c9dfc --- /dev/null +++ b/tests/packagedcode/data/pypi/pip-cache-origin-json/pefile/origin.json @@ -0,0 +1 @@ +{"archive_info":{"hash":"sha256=a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b","hashes":{"sha256":"a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b"}},"url":"https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz"} \ No newline at end of file diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 20afae813db..b16bc5ad39f 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -806,3 +806,66 @@ def test_parse_setup_py(test_loc): def test_parse_more_setup_py(test_loc): check_setup_py_parsing(test_loc) + + +class TestPipWheelCacheOriginJsonHandler(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_is_pip_wheel_cache_origin_json(self): + test_file = self.get_test_loc('pypi/pip-cache-origin-json/pefile/origin.json') + assert pypi.PipWheelCacheOriginJsonHandler.is_datafile(test_file) + + def test_parse_pip_wheel_cache_origin_json(self): + test_file = self.get_test_loc('pypi/pip-cache-origin-json/pefile/origin.json') + package = pypi.PipWheelCacheOriginJsonHandler.parse(test_file) + expected_loc = self.get_test_loc( + 'pypi/pip-cache-origin-json/pefile-origin.json-expected.json', + must_exist=False, + ) + self.check_packages_data( + package, expected_loc, must_exist=False, regen=REGEN_TEST_FIXTURES, + ) + + def test_parse_origin_json_missing_hash_does_not_fail(self): + test_file = self.get_test_loc('pypi/pip-cache-origin-json/missing-hash/origin.json') + packages = list(pypi.PipWheelCacheOriginJsonHandler.parse(test_file)) + assert len(packages) == 1 + package = packages[0].to_dict() + assert package['name'] == 'requests' + assert package['version'] == '2.32.3' + assert package['download_url'] == ( + 'https://files.pythonhosted.org/packages/63/70/' + '2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz' + ) + assert package['sha256'] is None + + def test_parse_origin_json_missing_url_uses_sibling_wheel(self): + test_file = self.get_test_loc('pypi/pip-cache-origin-json/missing-url/origin.json') + packages = list(pypi.PipWheelCacheOriginJsonHandler.parse(test_file)) + assert len(packages) == 1 + package = packages[0].to_dict() + assert package['name'] == 'pefile' + assert package['version'] == '2022.5.30' + assert package['download_url'] is None + assert package['sha256'] == ( + 'a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b' + ) + + def test_parse_origin_json_invalid_content_does_not_fail(self): + test_file = self.get_test_loc('pypi/pip-cache-origin-json/invalid/origin.json') + packages = list(pypi.PipWheelCacheOriginJsonHandler.parse(test_file)) + assert packages == [] + + def test_package_scan_pip_wheel_cache_origin_json_end_to_end(self): + test_dir = self.get_test_loc('pypi/pip-cache-origin-json/pefile/') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc( + 'pypi/pip-cache-origin-json/pefile-package-expected.json', + must_exist=False, + ) + run_scan_click( + ['--package', '--strip-root', '--processes', '-1', test_dir, '--json-pp', result_file] + ) + check_json_scan( + expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES, + )