Merge pull request #313 from gerrod3/default_ondemand

Sync default policy is on_demand.

Author: dralley

CHANGES/7401.misc

@@ -0,0 +1 @@
+Change the default download policy to 'on_demand', as bulk syncs of PyPI would use several terabytes of disk space.

docs/_scripts/remote.sh

@@ -2,7 +2,7 @@
 http POST $BASE_ADDR/pulp/api/v3/remotes/python/python/ \
     name='bar' \
     url='https://pypi.org/' \
-    includes:='[{"name": "shelf-reader"}]'
+    includes:='["shelf-reader"]'
 
 # Export an environment variable for the new remote URI.
 export REMOTE_HREF=$(http $BASE_ADDR/pulp/api/v3/remotes/python/python/ \

docs/workflows/sync.rst

@@ -38,24 +38,25 @@ itself, a fixture, or even an instance of Pulp 2.
 Remote GET Response::
 
     {
-        "pulp_created": "2019-04-29T15:58:01.196433Z",
-        "pulp_href": "/pulp/api/v3/remotes/python/python/1962b474-1545-4de1-adf4-4bf211679752/",
-        "pulp_last_updated": "2019-04-29T15:58:01.196446Z",
-        "download_concurrency": 20,
+        "ca_cert": null,
+        "client_cert": null,
+        "client_key": null,
+        "download_concurrency": 10,
         "excludes": [],
         "includes": [
-            {
-                "name": "shelf-reader",
-                "version_specifier": ""
-            }
+            "shelf-reader"
         ],
         "name": "bar",
-        "policy": "immediate",
+        "password": null,
+        "policy": "on_demand",
         "prereleases": false,
-        "proxy_url": "",
-        "ssl_validation": true,
+        "proxy_url": null,
+        "pulp_created": "2020-10-15T22:08:07.943369Z",
+        "pulp_href": "/pulp/api/v3/remotes/python/python/bbcfa980-ea53-425c-b03b-17c9c2867bba/",
+        "pulp_last_updated": "2020-10-15T22:08:07.943404Z",
+        "tls_validation": true,
         "url": "https://pypi.org/",
-        "validate": true
+        "username": null
     }
 
 Reference: `Python Remote Usage <../restapi.html#tag/remotes>`_
@@ -71,13 +72,10 @@ about will be synced::
         name='complex-remote' \
         url='https://pypi.org/' \
         includes:='[
-            { "name": "django",
-              "version_specifier": "~=2.0,!=2.0.1",
-            },
-            {"name": "pip-tools",
-             "version_specifier": ">=1.12,<=2.0"},
-            {"name": "scipy"},
-            {"name": "shelf-reader"}
+            "django~=2.0,!=2.0.1",
+            "pip-tools>=1.12,<=2.0",
+            "scipy",
+            "shelf-reader"
         ]'
 
 You can also use version specifiers to "exclude" certain versions of a project, like so::
@@ -86,16 +84,34 @@ You can also use version specifiers to "exclude" certain versions of a project,
         name='complex-remote' \
         url='https://pypi.org/' \
         includes:='[
-            {"name": "django", "version_specifier": ""},
-            {"name": "scipy", "version_specifier": ""}
+            "django",
+            "scipy"
         ]' \
         excludes:='[
-            {"name": "django", "version_specifier": "~=1.0"},
-            {"name": "scipy"}
+            "django~=1.0",
+            "scipy"
         ]'
 
 Reference: `Python Remote Usage <../restapi.html#tag/remotes>`_
 
+Creating a remote to sync all of PyPi
+_____________________________________
+
+A remote can be setup to sync all of PyPi by not specifying any included packages like so::
+
+    $ http POST $BASE_ADDR/pulp/api/v3/remotes/python/python/ \
+        name='PyPi-mirror' \
+        url='https://pypi.org/' \
+        excludes:='[
+            "django~=1.0",
+            "scipy"
+        ]'
+
+By not setting the "includes" field Pulp will ask PyPi for all of its available packages to sync, minus the ones from
+the excludes field. Default Python remotes are created with syncing policy "on_demand" because the most common
+Python remotes involve syncing with PyPi which requires terabytes of disk space. This can be changed by
+modifying the "policy" field.
+
 Sync repository foo with remote
 -------------------------------
 

pulp_python/app/serializers.py

@@ -265,7 +265,7 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer):
         help_text=_("The policy to use when downloading content. The possible values include: "
                     "'immediate', 'on_demand', and 'cache_only'. 'immediate' is the default."),
         choices=core_models.Remote.POLICY_CHOICES,
-        default=core_models.Remote.IMMEDIATE
+        default=core_models.Remote.ON_DEMAND
     )
 
     def validate_includes(self, value):
@@ -316,7 +316,7 @@ class PythonBanderRemoteSerializer(serializers.Serializer):
         help_text=_("The policy to use when downloading content. The possible values include: "
                     "'immediate', 'on_demand', and 'cache_only'. 'immediate' is the default."),
         choices=core_models.Remote.POLICY_CHOICES,
-        default=core_models.Remote.IMMEDIATE
+        default=core_models.Remote.ON_DEMAND
     )
 
 

pulp_python/tests/functional/api/test_crud_remotes.py

@@ -208,23 +208,23 @@ def tearDownClass(cls):
             cls.remote_api.delete(result["pulp_href"])
 
     def test_01_no_defined_policy(self):
-        """Verify the default policy `immediate`.
+        """Verify the default policy `on_demand`.
 
-        When no policy is defined, the default policy of `immediate`
+        When no policy is defined, the default policy of `on_demand`
         is applied.
         """
         del self.body["policy"]
         self.remote.update(self.remote_api.create(self.body).to_dict())
-        self.assertEqual(self.remote["policy"], "immediate", self.remote)
+        self.assertEqual(self.remote["policy"], "on_demand", self.remote)
 
     @skip_if(len, "policies", 1)
     def test_02_change_policy(self):
         """Verify ability to change policy to value other than the default.
 
-        Update the remote policy to a valid value other than `immedaite`
+        Update the remote policy to a valid value other than `on_demand`
         and verify the new set value.
         """
-        changed_policy = choice([item for item in self.policies if item != "immediate"])
+        changed_policy = choice([item for item in self.policies if item != "on_demand"])
         response = self.remote_api.partial_update(
             self.remote["pulp_href"], {"policy": changed_policy}
         )

pulp_python/tests/functional/constants.py

@@ -212,7 +212,7 @@
 DEFAULT_BANDER_REMOTE_BODY = {
     "url": "https://pypi.org",
     "download_concurrency": 3,
-    "policy": "immediate",
+    "policy": "on_demand",
     "prereleases": False,
     "excludes": ["example1", "example2"]
 }