diff --git a/CHANGES/689.feature b/CHANGES/689.feature new file mode 100644 index 00000000..99887af4 --- /dev/null +++ b/CHANGES/689.feature @@ -0,0 +1 @@ +Added full support for the latest core metadata (up to 2.4). diff --git a/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py b/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py new file mode 100644 index 00000000..f3773c7b --- /dev/null +++ b/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.19 on 2025-07-09 08:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('python', '0013_add_rbac_permissions'), + ] + + operations = [ + migrations.AddField( + model_name='pythonpackagecontent', + name='dynamic', + field=models.JSONField(default=list), + ), + migrations.AddField( + model_name='pythonpackagecontent', + name='license_expression', + field=models.TextField(default=''), + preserve_default=False, + ), + migrations.AddField( + model_name='pythonpackagecontent', + name='license_file', + field=models.JSONField(default=list), + ), + migrations.AddField( + model_name='pythonpackagecontent', + name='provides_extras', + field=models.JSONField(default=list), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 39e98d39..adfa0f99 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -140,49 +140,66 @@ class PythonPackageContent(Content): """ A Content Type representing Python's Distribution Package. - As defined in pep-0426 and pep-0345. + Core Metadata: + https://packaging.python.org/en/latest/specifications/core-metadata/ - https://www.python.org/dev/peps/pep-0491/ - https://www.python.org/dev/peps/pep-0345/ - """ - - PROTECTED_FROM_RECLAIM = False + Release metadata (JSON API): + https://docs.pypi.org/api/json/ - TYPE = "python" - repo_key_fields = ("filename",) - # Required metadata - filename = models.TextField(db_index=True) - packagetype = models.TextField(choices=PACKAGE_TYPES) - name = models.TextField() - name.register_lookup(NormalizeName) - version = models.TextField() - sha256 = models.CharField(db_index=True, max_length=64) - # Optional metadata - python_version = models.TextField() - metadata_version = models.TextField() - summary = models.TextField() - description = models.TextField() - keywords = models.TextField() - home_page = models.TextField() - download_url = models.TextField() + File Formats: + https://packaging.python.org/en/latest/specifications/source-distribution-format/ + https://packaging.python.org/en/latest/specifications/binary-distribution-format/ + """ + # Core metadata + # Version 1.0 author = models.TextField() author_email = models.TextField() + description = models.TextField() + home_page = models.TextField() # Deprecated in favour of Project-URL + keywords = models.TextField() + license = models.TextField() # Deprecated in favour of License-Expression + metadata_version = models.TextField() + name = models.TextField() + platform = models.TextField() + summary = models.TextField() + version = models.TextField() + # Version 1.1 + classifiers = models.JSONField(default=list) + download_url = models.TextField() # Deprecated in favour of Project-URL + supported_platform = models.TextField() + # Version 1.2 maintainer = models.TextField() maintainer_email = models.TextField() - license = models.TextField() - requires_python = models.TextField() + obsoletes_dist = models.JSONField(default=list) project_url = models.TextField() - platform = models.TextField() - supported_platform = models.TextField() - requires_dist = models.JSONField(default=list) + project_urls = models.JSONField(default=dict) provides_dist = models.JSONField(default=list) - obsoletes_dist = models.JSONField(default=list) requires_external = models.JSONField(default=list) - classifiers = models.JSONField(default=list) - project_urls = models.JSONField(default=dict) + requires_dist = models.JSONField(default=list) + requires_python = models.TextField() + # Version 2.1 description_content_type = models.TextField() - # Pulp Domains - _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) + provides_extras = models.JSONField(default=list) + # Version 2.2 + dynamic = models.JSONField(default=list) + # Version 2.4 + license_expression = models.TextField() + license_file = models.JSONField(default=list) + + # Release metadata + filename = models.TextField(db_index=True) + packagetype = models.TextField(choices=PACKAGE_TYPES) + python_version = models.TextField() + sha256 = models.CharField(db_index=True, max_length=64) + + # From pulpcore + PROTECTED_FROM_RECLAIM = False + TYPE = "python" + _pulp_domain = models.ForeignKey( + "core.Domain", default=get_domain_pk, on_delete=models.PROTECT + ) + name.register_lookup(NormalizeName) + repo_key_fields = ("filename",) @staticmethod def init_from_artifact_and_relative_path(artifact, relative_path): diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index dca90d27..b1ef7b9f 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -72,69 +72,69 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa """ A Serializer for PythonPackageContent. """ - - filename = serializers.CharField( - help_text=_('The name of the distribution package, usually of the format:' - ' {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}' - '-{platform tag}.{packagetype}'), - read_only=True, - ) - packagetype = serializers.CharField( - help_text=_('The type of the distribution package ' - '(e.g. sdist, bdist_wheel, bdist_egg, etc)'), - read_only=True, - ) - name = serializers.CharField( - help_text=_('The name of the python project.'), - read_only=True, - ) - version = serializers.CharField( - help_text=_('The packages version number.'), - read_only=True, - ) - sha256 = serializers.CharField( - default='', - help_text=_('The SHA256 digest of this package.'), - ) - metadata_version = serializers.CharField( - help_text=_('Version of the file format'), - read_only=True, + # Core metadata + # Version 1.0 + author = serializers.CharField( + required=False, allow_blank=True, + help_text=_('Text containing the author\'s name. Contact information can also be added,' + ' separated with newlines.') ) - summary = serializers.CharField( + author_email = serializers.CharField( required=False, allow_blank=True, - help_text=_('A one-line summary of what the package does.') + help_text=_('The author\'s e-mail address. ') ) description = serializers.CharField( required=False, allow_blank=True, help_text=_('A longer description of the package that can run to several paragraphs.') ) - description_content_type = serializers.CharField( + home_page = serializers.CharField( required=False, allow_blank=True, - help_text=_('A string stating the markup syntax (if any) used in the distribution’s' - ' description, so that tools can intelligently render the description.') + help_text=_('The URL for the package\'s home page.') ) keywords = serializers.CharField( required=False, allow_blank=True, help_text=_('Additional keywords to be used to assist searching for the ' 'package in a larger catalog.') ) - home_page = serializers.CharField( + license = serializers.CharField( required=False, allow_blank=True, - help_text=_('The URL for the package\'s home page.') + help_text=_('Text indicating the license covering the distribution') ) - download_url = serializers.CharField( + metadata_version = serializers.CharField( + help_text=_('Version of the file format'), + read_only=True, + ) + name = serializers.CharField( + help_text=_('The name of the python project.'), + read_only=True, + ) + platform = serializers.CharField( required=False, allow_blank=True, - help_text=_('Legacy field denoting the URL from which this package can be downloaded.') + help_text=_('A comma-separated list of platform specifications, ' + 'summarizing the operating systems supported by the package.') ) - author = serializers.CharField( + summary = serializers.CharField( required=False, allow_blank=True, - help_text=_('Text containing the author\'s name. Contact information can also be added,' - ' separated with newlines.') + help_text=_('A one-line summary of what the package does.') ) - author_email = serializers.CharField( + version = serializers.CharField( + help_text=_('The packages version number.'), + read_only=True, + ) + # Version 1.1 + classifiers = serializers.JSONField( + required=False, default=list, + help_text=_('A JSON list containing classification values for a Python package.') + ) + download_url = serializers.CharField( required=False, allow_blank=True, - help_text=_('The author\'s e-mail address. ') + help_text=_('Legacy field denoting the URL from which this package can be downloaded.') ) + supported_platform = serializers.CharField( + required=False, allow_blank=True, + help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') + ) + # Version 1.2 maintainer = serializers.CharField( required=False, allow_blank=True, help_text=_('The maintainer\'s name at a minimum; ' @@ -144,14 +144,11 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('The maintainer\'s e-mail address.') ) - license = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text indicating the license covering the distribution') - ) - requires_python = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The Python version(s) that the distribution is guaranteed to be ' - 'compatible with.') + obsoletes_dist = serializers.JSONField( + required=False, default=list, + help_text=_('A JSON list containing names of a distutils project\'s distribution which ' + 'this distribution renders obsolete, meaning that the two projects should not ' + 'be installed at the same time.') ) project_url = serializers.CharField( required=False, allow_blank=True, @@ -161,39 +158,73 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, default=dict, help_text=_('A dictionary of labels and URLs for the project.') ) - platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A comma-separated list of platform specifications, ' - 'summarizing the operating systems supported by the package.') + provides_dist = serializers.JSONField( + required=False, default=list, + help_text=_('A JSON list containing names of a Distutils project which is contained' + ' within this distribution.') ) - supported_platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') + requires_external = serializers.JSONField( + required=False, default=list, + help_text=_('A JSON list containing some dependency in the system that the distribution ' + 'is to be used.') ) requires_dist = serializers.JSONField( required=False, default=list, help_text=_('A JSON list containing names of some other distutils project ' 'required by this distribution.') ) - provides_dist = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of a Distutils project which is contained' - ' within this distribution.') + requires_python = serializers.CharField( + required=False, allow_blank=True, + help_text=_('The Python version(s) that the distribution is guaranteed to be ' + 'compatible with.') ) - obsoletes_dist = serializers.JSONField( + # Version 2.1 + description_content_type = serializers.CharField( + required=False, allow_blank=True, + help_text=_('A string stating the markup syntax (if any) used in the distribution’s' + ' description, so that tools can intelligently render the description.') + ) + provides_extras = serializers.JSONField( required=False, default=list, - help_text=_('A JSON list containing names of a distutils project\'s distribution which ' - 'this distribution renders obsolete, meaning that the two projects should not ' - 'be installed at the same time.') + help_text=_('A JSON list containing names of optional features provided by the package.') ) - requires_external = serializers.JSONField( + # Version 2.2 + dynamic = serializers.JSONField( required=False, default=list, - help_text=_('A JSON list containing some dependency in the system that the distribution ' - 'is to be used.') + help_text=_('A JSON list containing names of other core metadata fields which are ' + 'permitted to vary between sdist and bdist packages. Fields NOT marked ' + 'dynamic MUST be the same between bdist and sdist.') ) - classifiers = serializers.JSONField( + # Version 2.4 + license_expression = serializers.CharField( + required=False, allow_blank=True, + help_text=_('Text string that is a valid SPDX license expression.') + ) + license_file = serializers.JSONField( required=False, default=list, - help_text=_('A JSON list containing classification values for a Python package.') + help_text=_('A JSON list containing names of the paths to license-related files.') + ) + # Release metadata + filename = serializers.CharField( + help_text=_('The name of the distribution package, usually of the format:' + ' {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}' + '-{platform tag}.{packagetype}'), + read_only=True, + ) + packagetype = serializers.CharField( + help_text=_('The type of the distribution package ' + '(e.g. sdist, bdist_wheel, bdist_egg, etc)'), + read_only=True, + ) + python_version = serializers.CharField( + help_text=_( + 'The tag that indicates which Python implementation or version the package requires.' + ), + read_only=True, + ) + sha256 = serializers.CharField( + default='', + help_text=_('The SHA256 digest of this package.'), ) def deferred_validate(self, data): @@ -242,11 +273,13 @@ def retrieve(self, validated_data): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( - 'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary', - 'description', 'description_content_type', 'keywords', 'home_page', 'download_url', - 'author', 'author_email', 'maintainer', 'maintainer_email', 'license', - 'requires_python', 'project_url', 'project_urls', 'platform', 'supported_platform', - 'requires_dist', 'provides_dist', 'obsoletes_dist', 'requires_external', 'classifiers' + 'author', 'author_email', 'description', 'home_page', 'keywords', 'license', + 'metadata_version', 'name', 'platform', 'summary', 'version', 'classifiers', + 'download_url', 'supported_platform', 'maintainer', 'maintainer_email', + 'obsoletes_dist', 'project_url', 'project_urls', 'provides_dist', 'requires_external', + 'requires_dist', 'requires_python', 'description_content_type', + 'provides_extras', 'dynamic', 'license_expression', 'license_file', + 'filename', 'packagetype', 'python_version', 'sha256' ) model = python_models.PythonPackageContent diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 89d1c3b4..35458718 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -87,35 +87,46 @@ def parse_project_metadata(project): dictionary: of python project metadata """ - package = {} - package['name'] = project.get('name') or "" - package['version'] = project.get('version') or "" - package['packagetype'] = project.get('packagetype') or "" - package['metadata_version'] = project.get('metadata_version') or "" - package['summary'] = project.get('summary') or "" - package['description'] = project.get('description') or "" - package['keywords'] = project.get('keywords') or "" - package['home_page'] = project.get('home_page') or "" - package['download_url'] = project.get('download_url') or "" - package['author'] = project.get('author') or "" - package['author_email'] = project.get('author_email') or "" - package['maintainer'] = project.get('maintainer') or "" - package['maintainer_email'] = project.get('maintainer_email') or "" - package['license'] = project.get('license') or "" - package['project_url'] = project.get('project_url') or "" - package['platform'] = project.get('platform') or "" - package['supported_platform'] = project.get('supported_platform') or "" - package['requires_python'] = project.get('requires_python') or "" - package['requires_dist'] = json.dumps(project.get('requires_dist', [])) - package['provides_dist'] = json.dumps(project.get('provides_dist', [])) - package['obsoletes_dist'] = json.dumps(project.get('obsoletes_dist', [])) - package['requires_external'] = json.dumps(project.get('requires_external', [])) - package['classifiers'] = json.dumps(project.get('classifiers', [])) - package['project_urls'] = json.dumps(project.get('project_urls', {})) - package['description_content_type'] = project.get('description_content_type') or "" - package['python_version'] = project.get('python_version') or "" - - return package + return { + # Core metadata + # Version 1.0 + 'author': project.get('author') or "", + 'author_email': project.get('author_email') or "", + 'description': project.get('description') or "", + 'home_page': project.get('home_page') or "", + 'keywords': project.get('keywords') or "", + 'license': project.get('license') or "", + 'metadata_version': project.get('metadata_version') or "", + 'name': project.get('name') or "", + 'platform': project.get('platform') or "", + 'summary': project.get('summary') or "", + 'version': project.get('version') or "", + # Version 1.1 + 'classifiers': json.dumps(project.get('classifiers', [])), + 'download_url': project.get('download_url') or "", + 'supported_platform': project.get('supported_platform') or "", + # Version 1.2 + 'maintainer': project.get('maintainer') or "", + 'maintainer_email': project.get('maintainer_email') or "", + 'obsoletes_dist': json.dumps(project.get('obsoletes_dist', [])), + 'project_url': project.get('project_url') or "", + 'project_urls': json.dumps(project.get('project_urls', {})), + 'provides_dist': json.dumps(project.get('provides_dist', [])), + 'requires_external': json.dumps(project.get('requires_external', [])), + 'requires_dist': json.dumps(project.get('requires_dist', [])), + 'requires_python': project.get('requires_python') or "", + # Version 2.1 + 'description_content_type': project.get('description_content_type') or "", + 'provides_extras': json.dumps(project.get('provides_extras', [])), + # Version 2.2 + 'dynamic': json.dumps(project.get('dynamic', [])), + # Version 2.4 + 'license_expression': project.get('license_expression') or "", + 'license_file': json.dumps(project.get('license_file', [])), + # Release metadata + 'packagetype': project.get('packagetype') or "", + 'python_version': project.get('python_version') or "", + } def parse_metadata(project, version, distribution): @@ -311,6 +322,11 @@ def python_content_to_info(content): "classifiers": json_to_dict(content.classifiers) or None, "yanked": False, # These are no longer used on PyPI, but are still present "yanked_reason": None, + # New core metadata (Version 2.1, 2.2, 2.4) + "provides_extras": json_to_dict(content.provides_extras) or None, + "dynamic": json_to_dict(content.dynamic) or None, + "license_expression": content.license_expression or "", + "license_file": json_to_dict(content.license_file) or None, } diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 4264d051..648f27a8 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -111,6 +111,34 @@ def test_content_crud( assert msg in e.value.task.error["description"] +def test_content_create_new_metadata( + delete_orphans_pre, download_python_file, monitor_task, python_bindings +): + """ + Test the creation of python content unit with newly added core metadata (provides_extras, + dynamic, license_expression, license_file). + """ + python_egg_filename = "setuptools-80.9.0.tar.gz" + python_egg_url = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), python_egg_filename) + python_file = download_python_file(python_egg_filename, python_egg_url) + + body = {"relative_path": python_egg_filename, "file": python_file} + response = python_bindings.ContentPackagesApi.create(**body) + task = monitor_task(response.task) + content = python_bindings.ContentPackagesApi.read(task.created_resources[0]) + + python_package_data = { + "filename": "setuptools-80.9.0.tar.gz", + "provides_extras": + '["test", "doc", "ssl", "certs", "core", "check", "cover", "enabler", "type"]', + "dynamic": '["license-file"]', + "license_expression": "MIT", + "license_file": '["LICENSE"]', + } + for k, v in python_package_data.items(): + assert getattr(content, k) == v + + @pytest.mark.parallel def test_upload_metadata_23_spec(python_content_factory): """Test that packages using metadata spec 2.3 can be uploaded to pulp.""" @@ -139,11 +167,13 @@ def test_upload_requires_python(python_content_factory): @pytest.mark.parallel def test_upload_metadata_24_spec(python_content_factory): """Test that packages using metadata spec 2.4 can be uploaded to pulp.""" - filename = "urllib3-2.3.0-py3-none-any.whl" + filename = "setuptools-80.9.0.tar.gz" with PyPISimple() as client: - page = client.get_project_page("urllib3") + page = client.get_project_page("setuptools") for package in page.packages: if package.filename == filename: content = python_content_factory(filename, url=package.url) assert content.metadata_version == "2.4" + assert content.license_expression == "MIT" + assert content.license_file == '["LICENSE"]' break diff --git a/pulp_python/tests/functional/api/test_repair.py b/pulp_python/tests/functional/api/test_repair.py index 5a6930f5..cdfac9c6 100644 --- a/pulp_python/tests/functional/api/test_repair.py +++ b/pulp_python/tests/functional/api/test_repair.py @@ -96,8 +96,6 @@ def test_metadata_repair_command( } content = create_content_direct(python_file, data) for field, wrong_value in data.items(): - if field == "python_version": - continue assert getattr(content, field) == wrong_value move_to_repository(python_repo.pulp_href, [content.pulp_href]) diff --git a/pyproject.toml b/pyproject.toml index c975e336..22a04871 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ classifiers=[ requires-python = ">=3.9" dependencies = [ "pulpcore>=3.49.0,<3.85", - "pkginfo>=1.10.0,<1.13.0", + "pkginfo>=1.12.0,<1.13.0", "bandersnatch>=6.3.0,<6.4", # Anything >=6.4 requires Python 3.10+ "pypi-simple>=1.5.0,<2.0", ]