Add support for pull-through repository saving

fixes: #815
https://issues.redhat.com/browse/PULP-417

Author: gerrod3

CHANGES/815.feature

@@ -0,0 +1,2 @@
+Added support for automatically saving pull-through content to a repository.
+Requires pulpcore 3.74 or later.

pulp_python/app/models.py

@@ -275,6 +275,7 @@ class PythonRepository(Repository, AutoAddObjPermsMixin):
     TYPE = "python"
     CONTENT_TYPES = [PythonPackageContent]
     REMOTE_TYPES = [PythonRemote]
+    PULL_THROUGH_SUPPORTED = True
 
     autopublish = models.BooleanField(default=False)
 

pulp_python/app/pypi/views.py

@@ -275,8 +275,7 @@ def retrieve(self, request, path, package):
         # Should I redirect if the normalized name is different?
         normalized = canonicalize_name(package)
         if self.distribution.remote:
-            if not repo_ver or not content.filter(name__normalize=normalized).exists():
-                return self.pull_through_package_simple(normalized, path, self.distribution.remote)
+            return self.pull_through_package_simple(normalized, path, self.distribution.remote)
         if self.should_redirect(repo_version=repo_ver):
             return redirect(urljoin(self.base_content_url, f'{path}/simple/{normalized}/'))
         packages = (

pulp_python/tests/functional/api/test_full_mirror.py

@@ -1,7 +1,7 @@
 import pytest
 import requests
 import subprocess
-
+import time
 from pulp_python.tests.functional.constants import (
     PYPI_URL,
     PYTHON_XS_FIXTURE_CHECKSUMS,
@@ -12,6 +12,7 @@
 from pypi_simple import ProjectPage
 from packaging.version import parse
 from urllib.parse import urljoin, urlsplit
+from random import sample
 
 
 def test_pull_through_install(
@@ -97,17 +98,48 @@ def test_pull_through_filter(python_remote_factory, python_distribution_factory)
 
 @pytest.mark.parallel
 def test_pull_through_with_repo(
-    python_repo_with_sync, python_remote_factory, python_distribution_factory
+    python_repo_factory,
+    python_remote_factory,
+    python_distribution_factory,
+    python_bindings,
+    pulpcore_bindings,
+    monitor_task,
+    has_pulp_plugin,
 ):
-    """Tests that if content is already in repository, pull-through isn't used."""
-    remote = python_remote_factory()
-    repo = python_repo_with_sync(remote)
+    """Tests that content is saved to repository."""
+    remote = python_remote_factory(includes=[])
+    repo = python_repo_factory()
     distro = python_distribution_factory(repository=repo.pulp_href, remote=remote.pulp_href)
 
-    url = urljoin(distro.base_url, "simple/shelf-reader/")
-    project_page = ProjectPage.from_response(requests.get(url), "shelf-reader")
-
-    assert len(project_page.packages) == 2
-    for package in project_page.packages:
-        assert package.filename in PYTHON_XS_FIXTURE_CHECKSUMS
-        assert "?redirect=" not in package.url
+    # Perform a download of aiohttp to ensure it's saved to the repo
+    url = urljoin(distro.base_url, "simple/aiohttp/")
+    project_page = ProjectPage.from_response(requests.get(url), "aiohttp")
+    for package in sample(project_page.packages, 3):
+        assert "?redirect=" in package.url
+        r = requests.get(package.url)
+        assert r.status_code == 200
+
+    if has_pulp_plugin("core", max="3.73"):
+        pytest.skip("Pull-through repository save added in 3.74")
+
+    for _ in range(15):
+        tasks = pulpcore_bindings.TasksApi.list(reserved_resources=repo.prn)
+        if tasks.count == 3:
+            break
+        time.sleep(1)  # Wait for tasks to be created
+    else:
+        assert tasks.count == 3
+
+    for task in tasks.results:
+        monitor_task(task.pulp_href)
+
+    repo = python_bindings.RepositoriesPythonApi.read(repo.pulp_href)
+    assert repo.latest_version_href[-2] == "3"
+    content = python_bindings.ContentPackagesApi.list(repository_version=repo.latest_version_href)
+    assert content.count == 3
+
+    # Check that getting content that is already present doesn't trigger a task
+    r = requests.get(package.url)
+    assert r.status_code == 200
+    tasks = pulpcore_bindings.TasksApi.list(reserved_resources=repo.prn)
+    assert tasks.count == 3