From cfee40aa96ba32eb62b805a3563651f1c9c21e2a Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Wed, 29 Jan 2025 19:27:54 +0200 Subject: [PATCH 1/4] More porting from from requests to urllib3 This is related to #2762 (that replaces RequestsFetcher with Urllib3Fetcher) and takes care of the remaining requests use cases in the code base. Signed-off-by: Jussi Kukkonen --- examples/uploader/_localrepo.py | 11 ++++++----- tuf/__init__.py | 2 +- verify_release | 17 ++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/uploader/_localrepo.py b/examples/uploader/_localrepo.py index edae65821b..c4d746a34d 100644 --- a/examples/uploader/_localrepo.py +++ b/examples/uploader/_localrepo.py @@ -12,8 +12,8 @@ import os from datetime import datetime, timedelta, timezone -import requests from securesystemslib.signer import CryptoSigner, Signer +from urllib3 import request from tuf.api.exceptions import RepositoryError from tuf.api.metadata import Metadata, MetaFile, TargetFile, Targets @@ -92,8 +92,9 @@ def close(self, role_name: str, md: Metadata) -> None: # Upload using "api/role" uri = f"{self.base_url}/api/role/{role_name}" - r = requests.post(uri, data=md.to_bytes(JSONSerializer()), timeout=5) - r.raise_for_status() + r = request("POST", uri, body=md.to_bytes(JSONSerializer()), timeout=5) + if r.status != 200: + raise RuntimeError(f"HTTP error {r.status}") def add_target(self, role: str, targetpath: str) -> bool: """Add target to roles metadata and submit new metadata version""" @@ -124,8 +125,8 @@ def add_delegation(self, role: str) -> bool: data = {signer.public_key.keyid: signer.public_key.to_dict()} url = f"{self.base_url}/api/delegation/{role}" - r = requests.post(url, data=json.dumps(data), timeout=5) - if r.status_code != 200: + r = request("POST", url, body=json.dumps(data), timeout=5) + if r.status != 200: print(f"delegation failed with {r}") return False diff --git a/tuf/__init__.py b/tuf/__init__.py index b09503961c..4b25e51db8 100644 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -3,5 +3,5 @@ """TUF.""" -# This value is used in the requests user agent. +# This value is used in the ngclient user agent. __version__ = "5.1.0" diff --git a/verify_release b/verify_release index 549b7bab84..f6ae8330f0 100755 --- a/verify_release +++ b/verify_release @@ -10,7 +10,6 @@ on GitHub and PyPI match the built release artifacts. """ import argparse -import json import os import subprocess import sys @@ -20,10 +19,10 @@ from typing import Optional try: import build as _ # type: ignore[import-not-found] # noqa: F401 - import requests + from urllib3 import request except ImportError: - print("Error: verify_release requires modules 'requests' and 'build':") - print(" pip install requests build") + print("Error: verify_release requires modules 'urllib3' and 'build':") + print(" pip install urllib3 build") sys.exit(1) # Project variables @@ -75,9 +74,7 @@ def get_git_version() -> str: def get_github_version() -> str: """Return version string of latest GitHub release""" release_json = f"https://api.github.com/repos/{GITHUB_ORG}/{GITHUB_PROJECT}/releases/latest" - releases = json.loads( - requests.get(release_json, timeout=HTTP_TIMEOUT).content - ) + releases = request("GET", release_json, timeout=HTTP_TIMEOUT).json() return releases["tag_name"][1:] @@ -106,9 +103,11 @@ def verify_github_release(version: str, compare_dir: str) -> bool: with TemporaryDirectory() as github_dir: for filename in [tar, wheel]: url = f"{base_url}/v{version}/{filename}" - response = requests.get(url, stream=True, timeout=HTTP_TIMEOUT) + response = request( + "GET", url, preload_content=False, timeout=HTTP_TIMEOUT + ) with open(os.path.join(github_dir, filename), "wb") as f: - for data in response.iter_content(): + for data in response.stream(): f.write(data) return cmp( From 140abd34dc30507c18ddf9590e88b7aef30298b2 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Fri, 14 Feb 2025 16:17:29 +0200 Subject: [PATCH 2/4] Update requirements: drop requests This also removes RequestsFetcher from tuf.ngclient.__init__.py: Otherwise we can't drop the requests dependency. This means RequestsFetcher is not currently public. Signed-off-by: Jussi Kukkonen --- pyproject.toml | 1 - requirements/main.txt | 2 +- requirements/pinned.txt | 10 +--------- tuf/ngclient/__init__.py | 2 -- 4 files changed, 2 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4eac696d2c..1ea4a3365f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,6 @@ classifiers = [ "Topic :: Software Development", ] dependencies = [ - "requests>=2.19.1", "securesystemslib~=1.0", "urllib3<3,>=1.21.1", ] diff --git a/requirements/main.txt b/requirements/main.txt index e93071ff00..611c6589d8 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -7,4 +7,4 @@ # triggers CI/CD builds to automatically test against updated dependencies. # securesystemslib[crypto] -requests +urllib3 diff --git a/requirements/pinned.txt b/requirements/pinned.txt index e97461353f..038c8af1b0 100644 --- a/requirements/pinned.txt +++ b/requirements/pinned.txt @@ -4,21 +4,13 @@ # # pip-compile --output-file=requirements/pinned.txt --strip-extras requirements/main.txt # -certifi==2025.1.31 - # via requests cffi==1.17.1 # via cryptography -charset-normalizer==3.4.1 - # via requests cryptography==44.0.1 # via securesystemslib -idna==3.10 - # via requests pycparser==2.22 # via cffi -requests==2.32.3 - # via -r requirements/main.txt securesystemslib==1.2.0 # via -r requirements/main.txt urllib3==2.3.0 - # via requests + # via -r requirements/main.txt diff --git a/tuf/ngclient/__init__.py b/tuf/ngclient/__init__.py index 1d2084acf5..0c254e195a 100644 --- a/tuf/ngclient/__init__.py +++ b/tuf/ngclient/__init__.py @@ -8,7 +8,6 @@ # requests_fetcher is public but comes from _internal for now (because # sigstore-python 1.0 still uses the module from there). requests_fetcher # can be moved out of _internal once sigstore-python 1.0 is not relevant. -from tuf.ngclient._internal.requests_fetcher import RequestsFetcher from tuf.ngclient._internal.urllib3_fetcher import Urllib3Fetcher from tuf.ngclient.config import UpdaterConfig from tuf.ngclient.fetcher import FetcherInterface @@ -16,7 +15,6 @@ __all__ = [ # noqa: PLE0604 FetcherInterface.__name__, - RequestsFetcher.__name__, Urllib3Fetcher.__name__, TargetFile.__name__, Updater.__name__, From 6ddc0838a2acd44ac4869196d4fd4e909c4b9665 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Fri, 14 Feb 2025 16:31:28 +0200 Subject: [PATCH 3/4] Move fetchers around to make them public RequestsFetcher should still be public (even if deprecated). * We don't want to import RequestsFetcher in __init__ (because that requires importing requests) * but we do want RequestsFetcher to be importable publicly Move both fetchers out of _internal: that was never the right place for them anyway: they are public modules. Signed-off-by: Jussi Kukkonen --- pyproject.toml | 2 +- tuf/ngclient/__init__.py | 6 +----- tuf/ngclient/{_internal => }/requests_fetcher.py | 4 ---- tuf/ngclient/updater.py | 3 ++- tuf/ngclient/{_internal => }/urllib3_fetcher.py | 0 5 files changed, 4 insertions(+), 11 deletions(-) rename tuf/ngclient/{_internal => }/requests_fetcher.py (96%) rename tuf/ngclient/{_internal => }/urllib3_fetcher.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 1ea4a3365f..519739e5a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,4 +155,4 @@ exclude_also = [ ] [tool.coverage.run] branch = true -omit = [ "tests/*", "tuf/ngclient/_internal/requests_fetcher.py" ] +omit = [ "tests/*", "tuf/ngclient/requests_fetcher.py" ] diff --git a/tuf/ngclient/__init__.py b/tuf/ngclient/__init__.py index 0c254e195a..afab48f5cd 100644 --- a/tuf/ngclient/__init__.py +++ b/tuf/ngclient/__init__.py @@ -4,14 +4,10 @@ """TUF client public API.""" from tuf.api.metadata import TargetFile - -# requests_fetcher is public but comes from _internal for now (because -# sigstore-python 1.0 still uses the module from there). requests_fetcher -# can be moved out of _internal once sigstore-python 1.0 is not relevant. -from tuf.ngclient._internal.urllib3_fetcher import Urllib3Fetcher from tuf.ngclient.config import UpdaterConfig from tuf.ngclient.fetcher import FetcherInterface from tuf.ngclient.updater import Updater +from tuf.ngclient.urllib3_fetcher import Urllib3Fetcher __all__ = [ # noqa: PLE0604 FetcherInterface.__name__, diff --git a/tuf/ngclient/_internal/requests_fetcher.py b/tuf/ngclient/requests_fetcher.py similarity index 96% rename from tuf/ngclient/_internal/requests_fetcher.py rename to tuf/ngclient/requests_fetcher.py index 2f89e47ab4..99f32e759b 100644 --- a/tuf/ngclient/_internal/requests_fetcher.py +++ b/tuf/ngclient/requests_fetcher.py @@ -5,10 +5,6 @@ library. """ -# requests_fetcher is public but comes from _internal for now (because -# sigstore-python 1.0 still uses the module from there). requests_fetcher -# can be moved out of _internal once sigstore-python 1.0 is not relevant. - from __future__ import annotations import logging diff --git a/tuf/ngclient/updater.py b/tuf/ngclient/updater.py index 022d601f95..8c88a96ead 100644 --- a/tuf/ngclient/updater.py +++ b/tuf/ngclient/updater.py @@ -49,7 +49,8 @@ from tuf.api import exceptions from tuf.api.metadata import Root, Snapshot, TargetFile, Targets, Timestamp -from tuf.ngclient._internal import trusted_metadata_set, urllib3_fetcher +from tuf.ngclient import urllib3_fetcher +from tuf.ngclient._internal import trusted_metadata_set from tuf.ngclient.config import EnvelopeType, UpdaterConfig if TYPE_CHECKING: diff --git a/tuf/ngclient/_internal/urllib3_fetcher.py b/tuf/ngclient/urllib3_fetcher.py similarity index 100% rename from tuf/ngclient/_internal/urllib3_fetcher.py rename to tuf/ngclient/urllib3_fetcher.py From 5acd3f7df7ed1b0dd244621d79e2f2282e097d35 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Fri, 14 Feb 2025 17:36:11 +0200 Subject: [PATCH 4/4] ngclient: Add note about RequestsFetcher being deprecated Signed-off-by: Jussi Kukkonen --- tuf/ngclient/requests_fetcher.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tuf/ngclient/requests_fetcher.py b/tuf/ngclient/requests_fetcher.py index 99f32e759b..6edc699d9d 100644 --- a/tuf/ngclient/requests_fetcher.py +++ b/tuf/ngclient/requests_fetcher.py @@ -3,6 +3,13 @@ """Provides an implementation of ``FetcherInterface`` using the Requests HTTP library. + +Note that this module is deprecated, and the default fetcher is +Urllib3Fetcher: +* RequestsFetcher is still available to make it easy to fall back to + previous implementation if issues are found with the Urllib3Fetcher +* If RequestsFetcher is used, note that `requests` must be explicitly + depended on: python-tuf does not do that. """ from __future__ import annotations