Merge pull request #335 from gerrod3/new_sync_filters

Added new sync filtering options

Author: gerrod3

CHANGES/339.feature

@@ -0,0 +1 @@
+Added new sync filter `keep_latest_packages` to specify how many latest versions of packages to sync

CHANGES/341.feature

@@ -0,0 +1 @@
+Added new sync filters `package_types` and `exclude_platforms` to specify package types to sync

docs/workflows/sync.rst

@@ -104,6 +104,17 @@ You can also use version specifiers to "exclude" certain versions of a project,
             "scipy"
         ]'
 
+You can also filter packages by their type, platform and amount synced through the "package_types",
+"exclude_platforms", and "keep_latest_packages" fields respectively, like so::
+
+    $ pulp python remote create \
+        --name 'complex-filters' \
+        --url 'https://pypi.org/' \
+        --includes '["django"]' \
+        --package-types '["sdist", "bdist-wheel"]' # only sync sdist and bdist-wheel package types \
+        --exclude-platforms '["windows"]' # exclude any packages built for windows \
+        --keep-latest-packages 5 # keep the five latest versions
+
 Reference: `Python Remote Usage <../restapi.html#tag/remotes>`_
 
 .. _mirror-workflow:

pulp_python/app/migrations/0003_new_sync_filters.py

@@ -0,0 +1,29 @@
+# Generated by Django 2.2.19 on 2021-03-26 13:31
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('python', '0002_pythonpackagecontent_python_version'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pythonremote',
+            name='exclude_platforms',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=10), choices=[('windows', 'windows'), ('macos', 'macos'), ('freebsd', 'freebsd'), ('linux', 'linux')], default=list, size=None),
+        ),
+        migrations.AddField(
+            model_name='pythonremote',
+            name='keep_latest_packages',
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='pythonremote',
+            name='package_types',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=15), choices=[('bdist_dmg', 'bdist_dmg'), ('bdist_dumb', 'bdist_dumb'), ('bdist_egg', 'bdist_egg'), ('bdist_msi', 'bdist_msi'), ('bdist_rpm', 'bdist_rpm'), ('bdist_wheel', 'bdist_wheel'), ('bdist_wininst', 'bdist_wininst'), ('sdist', 'sdist')], default=list, size=None),
+        ),
+    ]

pulp_python/app/models.py

@@ -1,7 +1,7 @@
 from logging import getLogger
 
 from aiohttp.web import json_response
-from django.contrib.postgres.fields import JSONField
+from django.contrib.postgres.fields import ArrayField, JSONField
 from django.db import models
 
 from pulpcore.plugin.models import (
@@ -29,6 +29,11 @@
     ("sdist", "sdist"),
 )
 
+PLATFORMS = (("windows", "windows"),
+             ("macos", "macos"),
+             ("freebsd", "freebsd"),
+             ("linux", "linux"))
+
 
 class PythonDistribution(PublicationDistribution):
     """
@@ -157,6 +162,11 @@ class PythonRemote(Remote):
     prereleases = models.BooleanField(default=False)
     includes = JSONField(default=list)
     excludes = JSONField(default=list)
+    package_types = ArrayField(models.CharField(max_length=15, blank=True),
+                               choices=PACKAGE_TYPES, default=list)
+    keep_latest_packages = models.IntegerField(default=0)
+    exclude_platforms = ArrayField(models.CharField(max_length=10, blank=True),
+                                   choices=PLATFORMS, default=list)
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_python/app/serializers.py

@@ -237,6 +237,16 @@ class Meta:
         model = python_models.PythonPackageContent
 
 
+class MultipleChoiceArrayField(serializers.MultipleChoiceField):
+    """
+    A wrapper to make sure this DRF serializer works properly with ArrayFields.
+    """
+
+    def to_internal_value(self, data):
+        """Converts set to list."""
+        return list(super().to_internal_value(data))
+
+
 class PythonRemoteSerializer(core_serializers.RemoteSerializer):
     """
     A Serializer for PythonRemote.
@@ -267,6 +277,26 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
         choices=core_models.Remote.POLICY_CHOICES,
         default=core_models.Remote.ON_DEMAND
     )
+    package_types = MultipleChoiceArrayField(
+        required=False,
+        help_text=_("The package types to sync for Python content. Leave blank to get every"
+                    "package type."),
+        choices=python_models.PACKAGE_TYPES,
+        default=list
+    )
+    keep_latest_packages = serializers.IntegerField(
+        required=False,
+        help_text=_("The amount of latest versions of a package to keep on sync, includes"
+                    "pre-releases if synced. Default 0 keeps all versions."),
+        default=0
+    )
+    exclude_platforms = MultipleChoiceArrayField(
+        required=False,
+        help_text=_("List of platforms to exclude syncing Python packages for. Possible values"
+                    "include: windows, macos, freebsd, and linux."),
+        choices=python_models.PLATFORMS,
+        default=list
+    )
 
     def validate_includes(self, value):
         """Validates the includes"""
@@ -292,7 +322,8 @@ def validate_excludes(self, value):
 
     class Meta:
         fields = core_serializers.RemoteSerializer.Meta.fields + (
-            "includes", "excludes", "prereleases"
+            "includes", "excludes", "prereleases", "package_types", "keep_latest_packages",
+            "exclude_platforms",
         )
         model = python_models.PythonRemote
 

pulp_python/app/tasks/sync.py

@@ -74,6 +74,22 @@ def create_bandersnatch_config(remote):
         config["blocklist"]["packages"] = "\n".join(remote.excludes)
     if not remote.prereleases:
         config["plugins"]["enabled"] += "prerelease_release\n"
+    if remote.package_types:
+        rrfm = "regex_release_file_metadata"
+        config["plugins"]["enabled"] += rrfm
+        if not config.has_section(rrfm):
+            config.add_section(rrfm)
+        config[rrfm]["any:release_file.packagetype"] = "\n".join(remote.package_types)
+    if remote.keep_latest_packages:
+        config["plugins"]["enabled"] += "latest_release\n"
+        if not config.has_section("latest_release"):
+            config.add_section("latest_release")
+        config["latest_release"]["keep"] = str(remote.keep_latest_packages)
+    if remote.exclude_platforms:
+        config["plugins"]["enabled"] += "exclude_platform\n"
+        if not config.has_section("blocklist"):
+            config.add_section("blocklist")
+        config["blocklist"]["platforms"] = "\n".join(remote.exclude_platforms)
 
 
 class PythonBanderStage(Stage):

pulp_python/app/viewsets.py

@@ -160,12 +160,22 @@ def from_bandersnatch(self, request):
         enabled = bander_config.get("plugins", "enabled")
         enabled_all = "all" in enabled
         data["prereleases"] = not (enabled_all or "prerelease_release" in enabled)
-        if bander_config.has_option("allowlist", "packages") and \
-                (enabled_all or "allowlist_project" in enabled):
-            data["includes"] = bander_config.get("allowlist", "packages").split()
-        if bander_config.has_option("blocklist", "packages") and \
-                (enabled_all or "blocklist_project" in enabled):
-            data["excludes"] = bander_config.get("blocklist", "packages").split()
+        # TODO refactor to use a translation object
+        plugin_filters = {  # plugin : (section_name, bander_option, pulp_option)
+            "allowlist_project": ("allowlist", "packages", "includes"),
+            "blocklist_project": ("blocklist", "packages", "excludes"),
+            "regex_release_file_metadata": (
+                "regex_release_file_metadata",
+                "any:release_file.packagetype",
+                "package_types",
+            ),
+            "latest_release": ("latest_release", "keep", "keep_latest_packages"),
+            "exclude_platform": ("blocklist", "platforms", "exclude_platforms"),
+        }
+        for plugin, options in plugin_filters.items():
+            if (enabled_all or plugin in enabled) and \
+                    bander_config.has_option(options[0], options[1]):
+                data[options[2]] = bander_config.get(options[0], options[1]).split()
         remote = python_serializers.PythonRemoteSerializer(data=data, context={"request": request})
         remote.is_valid(raise_exception=True)
         remote.save()