diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index d3c48b6e259..66de54ae33f 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -172,6 +172,7 @@ # pypi.PypiSdistArchiveHandler, pypi.PypiWheelHandler, pypi.PyprojectTomlHandler, + pypi.PylockTomlHandler, pypi.PoetryPyprojectTomlHandler, pypi.PoetryLockHandler, pypi.PythonEditableInstallationPkgInfoFile, diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index b5588ed7ca9..c259c0452ac 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -597,6 +597,102 @@ def assemble(cls, package_data, resource, codebase, package_adder): yield lock_file +class PylockTomlHandler(models.DatafileHandler): + datasource_id = 'pypi_pylock_toml' + path_patterns = ('*pylock.toml',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python pylock.toml' + documentation_url = 'https://peps.python.org/pep-0751/' + + @classmethod + def parse(cls, location, package_only=False): + yield cls.build_package(location, package_only=package_only) + + @classmethod + def build_package(cls, location, package_only=False): + import tomllib + from packageurl import PackageURL + + with open(location, 'rb') as fp: + try: + lock_data = tomllib.load(fp) + except Exception as e: + logger_debug(f'PylockTomlHandler.parse: Error parsing TOML: {e}') + return None + + packages = lock_data.get('packages', []) + dependencies = [] + + for pkg in packages: + name = pkg.get('name') + version = pkg.get('version') + + deps = pkg.get('dependencies', []) + dependencies_for_resolved = [] + for dep in deps: + dep_name = dep.get('name') + if not dep_name: + continue + dep_purl = PackageURL( + type=cls.default_package_type, + name=dep_name, + ) + dependent_pkg = models.DependentPackage( + purl=dep_purl.to_string(), + scope='dependencies', + is_runtime=True, + is_optional=False, + is_direct=True, + is_pinned=False, + ) + dependencies_for_resolved.append(dependent_pkg.to_dict()) + + pkg_purl = PackageURL( + type=cls.default_package_type, + name=name, + version=version, + ) + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + name=name, + version=version, + is_virtual=True, + dependencies=dependencies_for_resolved, + ) + resolved_package = models.PackageData.from_data(package_data, package_only) + + dependency = models.DependentPackage( + purl=resolved_package.purl, + extracted_requirement=None, # In theory, pylock doesn't directly specify version specs in dependencies, they map directly to resolved names + scope='dependencies', + is_runtime=True, + is_optional=False, + is_direct=False, + is_pinned=True, # It is a locked package + resolved_package=resolved_package.to_dict(), + ) + dependencies.append(dependency.to_dict()) + + extra_data = {} + if lock_data.get('lock-version'): + extra_data['lock_version'] = lock_data.get('lock-version') + if lock_data.get('requires-python'): + extra_data['requires_python'] = lock_data.get('requires-python') + + env_package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language=cls.default_primary_language, + name='pylock-environment', + dependencies=dependencies, + extra_data=extra_data, + ) + return models.PackageData.from_data(env_package_data, package_only) + class PoetryPyprojectTomlHandler(BasePoetryPythonLayout): datasource_id = 'pypi_poetry_pyproject_toml' path_patterns = ('*pyproject.toml',) diff --git a/tests/packagedcode/data/plugin/plugins_list_linux.txt b/tests/packagedcode/data/plugin/plugins_list_linux.txt index eb4763d6c7e..697ddf75b26 100755 --- a/tests/packagedcode/data/plugin/plugins_list_linux.txt +++ b/tests/packagedcode/data/plugin/plugins_list_linux.txt @@ -762,6 +762,13 @@ Package type: pypi description: Python poetry pyproject.toml path_patterns: '*pyproject.toml' -------------------------------------------- +Package type: pypi + datasource_id: pypi_pylock_toml + documentation URL: https://peps.python.org/pep-0751/ + primary language: Python + description: Python pylock.toml + path_patterns: '*pylock.toml' +-------------------------------------------- Package type: pypi datasource_id: pypi_pyproject_toml documentation URL: https://packaging.python.org/en/latest/specifications/pyproject-toml/ diff --git a/tests/packagedcode/data/pypi/pylock/pylock.toml b/tests/packagedcode/data/pypi/pylock/pylock.toml new file mode 100644 index 00000000000..720524051d3 --- /dev/null +++ b/tests/packagedcode/data/pypi/pylock/pylock.toml @@ -0,0 +1,22 @@ +lock-version = '1.0' +environments = ["sys_platform == 'win32'", "sys_platform == 'linux'"] +requires-python = '==3.12' +created-by = 'mousebender' + +[[packages]] +name = 'attrs' +version = '25.1.0' +requires-python = '>=3.8' + +[[packages]] +name = 'cattrs' +version = '24.1.2' +requires-python = '>=3.8' +dependencies = [ + {name = 'attrs'}, +] + +[[packages]] +name = 'numpy' +version = '2.2.3' +requires-python = '>=3.10' diff --git a/tests/packagedcode/data/pypi/pylock/pylock.toml-expected.json b/tests/packagedcode/data/pypi/pylock/pylock.toml-expected.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/packagedcode/data/pypi/pylock/pylock.toml.expected b/tests/packagedcode/data/pypi/pylock/pylock.toml.expected new file mode 100644 index 00000000000..02deff05f13 --- /dev/null +++ b/tests/packagedcode/data/pypi/pylock/pylock.toml.expected @@ -0,0 +1,224 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "pylock-environment", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "lock_version": "1.0", + "requires_python": "==3.12" + }, + "dependencies": [ + { + "purl": "pkg:pypi/attrs@25.1.0", + "extracted_requirement": null, + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "attrs", + "version": "25.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/attrs@25.1.0" + }, + "extra_data": {} + }, + { + "purl": "pkg:pypi/cattrs@24.1.2", + "extracted_requirement": null, + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "cattrs", + "version": "24.1.2", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pypi/attrs", + "extracted_requirement": null, + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/cattrs@24.1.2" + }, + "extra_data": {} + }, + { + "purl": "pkg:pypi/numpy@2.2.3", + "extracted_requirement": null, + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "numpy", + "version": "2.2.3", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/numpy@2.2.3" + }, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/pylock-environment" + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_cargo.py b/tests/packagedcode/test_cargo.py index 2f6a2baa796..7519036b305 100644 --- a/tests/packagedcode/test_cargo.py +++ b/tests/packagedcode/test_cargo.py @@ -165,6 +165,7 @@ def test_scan_works_on_cargo_workspace_boring(self): ) def test_scan_works_on_rust_binary_with_inspector(self): + pytest.importorskip("rust_inspector") test_file = self.get_test_loc('cargo/binary/cargo_dependencies') expected_file = self.get_test_loc('cargo/binary/cargo-binary.expected.json') result_file = self.get_temp_file('results.json') diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 3dcfa7d4268..14485def1e9 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -364,6 +364,8 @@ def test_parse_pyproject_toml_standard_lc0(self): self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + class TestPoetryHandler(PackageTester): def test_is_pyproject_toml_poetry(self): @@ -404,6 +406,14 @@ def test_parse_pyproject_toml_poetry_univers(self): expected_loc = self.get_test_loc('pypi/poetry/univers-pyproject.toml-expected.json') self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) +class TestPylockTomlHandler(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse_pylock_toml(self): + test_file = self.get_test_loc('pypi/pylock/pylock.toml') + expected_loc = self.get_test_loc('pypi/pylock/pylock.toml.expected') + package = pypi.PylockTomlHandler.parse(test_file) + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) class TestPipInspectDeplockHandler(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data')