From 3eca8bda4075484592df75d82aff694f2407b97a Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 07:02:26 +0200 Subject: [PATCH 1/9] add support for caching in Redis --- httpx_caching/__init__.py | 2 ++ httpx_caching/__version__.py | 2 +- httpx_caching/_async/_cache_redis.py | 34 ++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 httpx_caching/_async/_cache_redis.py diff --git a/httpx_caching/__init__.py b/httpx_caching/__init__.py index c459097..ceab8ee 100644 --- a/httpx_caching/__init__.py +++ b/httpx_caching/__init__.py @@ -1,4 +1,5 @@ from httpx_caching._async._cache import AsyncDictCache +from httpx_caching._async._cache_redis import AsyncRedisCache from httpx_caching._async._transport import AsyncCachingTransport from httpx_caching._sync._cache import SyncDictCache from httpx_caching._sync._transport import SyncCachingTransport @@ -15,6 +16,7 @@ "SyncCachingTransport", "CachingClient", "AsyncDictCache", + "AsyncRedisCache", "SyncDictCache", "ExpiresAfterHeuristic", "LastModifiedHeuristic", diff --git a/httpx_caching/__version__.py b/httpx_caching/__version__.py index 92b8996..35b3a96 100644 --- a/httpx_caching/__version__.py +++ b/httpx_caching/__version__.py @@ -1,3 +1,3 @@ __title__ = "httpx-caching" __description__ = "Caching for HTTPX." -__version__ = "0.1a2" +__version__ = "0.1a2-redis" diff --git a/httpx_caching/_async/_cache_redis.py b/httpx_caching/_async/_cache_redis.py new file mode 100644 index 0000000..8de9ee0 --- /dev/null +++ b/httpx_caching/_async/_cache_redis.py @@ -0,0 +1,34 @@ +from typing import Optional, Tuple +import redis.asyncio as redis +from httpx_caching._models import Response +from httpx_caching._serializer import Serializer +from httpx_caching._utils import AsyncLock + + +class AsyncRedisCache: + def __init__(self, serializer: Optional[Serializer] = None) -> None: + self.serializer = serializer if serializer else Serializer() + self.redis = redis.Redis() + self.lock = None + + def get_lock(self): + if not self.lock: + self.lock = AsyncLock() + return self.lock + + async def aget(self, key: str) -> Tuple[Optional[Response], Optional[dict]]: + value = await self.redis.get(key) + return self.serializer.loads(value) + + async def aset( + self, key: str, response: Response, vary_header_data: dict, response_body: bytes + ) -> None: + async with self.get_lock(): + await self.redis.set(key, self.serializer.dumps(response, vary_header_data, response_body)) + + async def adelete(self, key: str) -> None: + async with self.get_lock(): + self.redis.delete(key) + + async def aclose(self): + await self.redis.close() diff --git a/setup.py b/setup.py index f28d013..98787cc 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ def get_packages(package): "msgpack", "anyio", "multimethod", + "redis" ], extras_require={}, classifiers=[ From b9e48d902117b5364444bbacc58f60a815af4078 Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 07:41:50 +0200 Subject: [PATCH 2/9] revert version to that of upstream --- httpx_caching/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpx_caching/__version__.py b/httpx_caching/__version__.py index 35b3a96..92b8996 100644 --- a/httpx_caching/__version__.py +++ b/httpx_caching/__version__.py @@ -1,3 +1,3 @@ __title__ = "httpx-caching" __description__ = "Caching for HTTPX." -__version__ = "0.1a2-redis" +__version__ = "0.1a2" From ac6f10fd8b5f101e922c30b25a7cb3f9203a154c Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 08:11:27 +0200 Subject: [PATCH 3/9] upgrade to httpx 0.23 --- setup.py | 2 +- tests/_async/test_client_actions.py | 4 ++-- tests/_async/test_etag.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 98787cc..a648a8e 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def get_packages(package): include_package_data=True, zip_safe=False, install_requires=[ - "httpx==0.22.*", + "httpx==0.23.*", "msgpack", "anyio", "multimethod", diff --git a/tests/_async/test_client_actions.py b/tests/_async/test_client_actions.py index d354486..af2b366 100644 --- a/tests/_async/test_client_actions.py +++ b/tests/_async/test_client_actions.py @@ -5,7 +5,7 @@ import mock import pytest -from httpx_caching import AsyncDictCache +from httpx_caching import AsyncDictCache, AsyncRedisCache from tests.conftest import cache_hit @@ -40,7 +40,7 @@ async def test_delete_invalidates_cache(self, url, async_client): @pytest.mark.xfail async def test_close(self, url, async_client): - mock_cache = mock.Mock(spec=AsyncDictCache) + mock_cache = mock.Mock(spec=AsyncRedisCache) async_client._transport.cache = mock_cache # TODO: httpx does not close transport if nothing has been done with the client diff --git a/tests/_async/test_etag.py b/tests/_async/test_etag.py index b07e896..fc84374 100644 --- a/tests/_async/test_etag.py +++ b/tests/_async/test_etag.py @@ -7,7 +7,7 @@ from freezegun import freeze_time from httpx import AsyncClient, Limits, Timeout -from httpx_caching import AsyncCachingTransport, AsyncDictCache +from httpx_caching import AsyncCachingTransport, AsyncDictCache, AsyncRedisCache from tests.conftest import cache_hit, raw_resp @@ -20,7 +20,7 @@ async def async_client(mocker): async_client = AsyncClient() transport = AsyncCachingTransport( transport=async_client._transport, - cache=AsyncDictCache(), + cache=AsyncRedisCache(), ) async_client._transport = transport From d784444c08953e98c1a4a7ae7a6a5af95aa366b9 Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 11:21:41 +0200 Subject: [PATCH 4/9] fix bug --- httpx_caching/_async/_cache_redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpx_caching/_async/_cache_redis.py b/httpx_caching/_async/_cache_redis.py index 8de9ee0..1cf6f0a 100644 --- a/httpx_caching/_async/_cache_redis.py +++ b/httpx_caching/_async/_cache_redis.py @@ -28,7 +28,7 @@ async def aset( async def adelete(self, key: str) -> None: async with self.get_lock(): - self.redis.delete(key) + await self.redis.delete(key) async def aclose(self): await self.redis.close() From 58ac3ae0c7325d020b351e2f1208bf4076629877 Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 11:21:49 +0200 Subject: [PATCH 5/9] update docs --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 3f17748..c9a5a86 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,11 @@ loop = asyncio.get_event_loop() loop.run_until_complete(run_example()) ``` +Or with Redis as cache backend: + +```python +client = CachingClient(client, cache=AsyncRedisCache()) +``` **Documentation:** From 0de480fb8189e0d397b4b45a655314ecb66b3281 Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 11:38:30 +0200 Subject: [PATCH 6/9] This changes were never intented to be commited --- tests/_async/test_client_actions.py | 4 ++-- tests/_async/test_etag.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/_async/test_client_actions.py b/tests/_async/test_client_actions.py index af2b366..d354486 100644 --- a/tests/_async/test_client_actions.py +++ b/tests/_async/test_client_actions.py @@ -5,7 +5,7 @@ import mock import pytest -from httpx_caching import AsyncDictCache, AsyncRedisCache +from httpx_caching import AsyncDictCache from tests.conftest import cache_hit @@ -40,7 +40,7 @@ async def test_delete_invalidates_cache(self, url, async_client): @pytest.mark.xfail async def test_close(self, url, async_client): - mock_cache = mock.Mock(spec=AsyncRedisCache) + mock_cache = mock.Mock(spec=AsyncDictCache) async_client._transport.cache = mock_cache # TODO: httpx does not close transport if nothing has been done with the client diff --git a/tests/_async/test_etag.py b/tests/_async/test_etag.py index fc84374..b07e896 100644 --- a/tests/_async/test_etag.py +++ b/tests/_async/test_etag.py @@ -7,7 +7,7 @@ from freezegun import freeze_time from httpx import AsyncClient, Limits, Timeout -from httpx_caching import AsyncCachingTransport, AsyncDictCache, AsyncRedisCache +from httpx_caching import AsyncCachingTransport, AsyncDictCache from tests.conftest import cache_hit, raw_resp @@ -20,7 +20,7 @@ async def async_client(mocker): async_client = AsyncClient() transport = AsyncCachingTransport( transport=async_client._transport, - cache=AsyncRedisCache(), + cache=AsyncDictCache(), ) async_client._transport = transport From 10f654a697bfee5ad8344aad06c52e15354eb00d Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 11:38:48 +0200 Subject: [PATCH 7/9] add vscode settings --- .vscode/settings.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9b38853 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file From e4cb9248298756aa848b332cf7617daff3e9d6a0 Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 11:39:09 +0200 Subject: [PATCH 8/9] Add sync version of the Redis cache adapter --- gen_sync.py | 1 + httpx_caching/__init__.py | 2 ++ httpx_caching/_sync/_cache_redis.py | 34 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 httpx_caching/_sync/_cache_redis.py diff --git a/gen_sync.py b/gen_sync.py index a33e456..4af277d 100644 --- a/gen_sync.py +++ b/gen_sync.py @@ -20,6 +20,7 @@ additional_replacements={ "AsyncBaseTransport": "BaseTransport", "AsyncHTTPTransport": "HTTPTransport", + "AsyncRedisCache": "SyncRedisCache", "async_client": "client", "AsyncClient": "Client", "make_async_client": "make_client", diff --git a/httpx_caching/__init__.py b/httpx_caching/__init__.py index ceab8ee..ff26612 100644 --- a/httpx_caching/__init__.py +++ b/httpx_caching/__init__.py @@ -2,6 +2,7 @@ from httpx_caching._async._cache_redis import AsyncRedisCache from httpx_caching._async._transport import AsyncCachingTransport from httpx_caching._sync._cache import SyncDictCache +from httpx_caching._sync._cache_redis import SyncRedisCache from httpx_caching._sync._transport import SyncCachingTransport from httpx_caching._wrapper import CachingClient @@ -18,6 +19,7 @@ "AsyncDictCache", "AsyncRedisCache", "SyncDictCache", + "SyncRedisCache", "ExpiresAfterHeuristic", "LastModifiedHeuristic", "OneDayCacheHeuristic", diff --git a/httpx_caching/_sync/_cache_redis.py b/httpx_caching/_sync/_cache_redis.py new file mode 100644 index 0000000..3fe22f4 --- /dev/null +++ b/httpx_caching/_sync/_cache_redis.py @@ -0,0 +1,34 @@ +from typing import Optional, Tuple +import redis as redis +from httpx_caching._models import Response +from httpx_caching._serializer import Serializer +from httpx_caching._utils import SyncLock + + +class SyncRedisCache: + def __init__(self, serializer: Optional[Serializer] = None) -> None: + self.serializer = serializer if serializer else Serializer() + self.redis = redis.Redis() + self.lock = None + + def get_lock(self): + if not self.lock: + self.lock = SyncLock() + return self.lock + + def get(self, key: str) -> Tuple[Optional[Response], Optional[dict]]: + value = self.redis.get(key) + return self.serializer.loads(value) + + def set( + self, key: str, response: Response, vary_header_data: dict, response_body: bytes + ) -> None: + with self.get_lock(): + self.redis.set(key, self.serializer.dumps(response, vary_header_data, response_body)) + + def delete(self, key: str) -> None: + with self.get_lock(): + self.redis.delete(key) + + def close(self): + self.redis.close() From 2c6160a935611fb21c49d62258ad18e81f579dec Mon Sep 17 00:00:00 2001 From: kovan Date: Sat, 8 Oct 2022 12:15:23 +0200 Subject: [PATCH 9/9] refactor to allow testing with custom cache class --- tests/_async/test_vary.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/_async/test_vary.py b/tests/_async/test_vary.py index 03e30d4..dc4ead9 100644 --- a/tests/_async/test_vary.py +++ b/tests/_async/test_vary.py @@ -3,10 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 from urllib.parse import urljoin +from httpx import AsyncClient import pytest -from httpx_caching import AsyncDictCache +from httpx_caching import AsyncDictCache, CachingClient from tests.conftest import cache_hit, make_async_client @@ -17,7 +18,8 @@ def cache(self): @pytest.fixture() async def async_client(self, cache): - async_client = make_async_client(cache=cache) + async_client = AsyncClient() + async_client = CachingClient(async_client, cache=cache) yield async_client await async_client.aclose()