Use the current highwater mark for config fetches

Author: jkebinger

prefab_cloud_python/_requests.py

@@ -73,15 +73,18 @@ def __next__(self):
         self.index = (self.index + 1) % len(self.hosts)
         return host
 
+
 # --- Simple LRU Cache Implementation ---
 
+
 @dataclass
 class CacheEntry:
     data: bytes
     etag: str
     expires_at: float
     url: str  # The full URL from the successful response
 
+
 class LRUCache:
     def __init__(self, max_size: int):
         self.max_size = max_size
@@ -120,9 +123,11 @@ def __init__(self, options):
         self.session = requests.Session()
         self.session.mount("https://", requests.adapters.HTTPAdapter())
         self.session.mount("http://", requests.adapters.HTTPAdapter())
-        self.session.headers.update({
-            "X-PrefabCloud-Client-Version": f"prefab-cloud-python-{getattr(options, 'version', 'development')}"
-        })
+        self.session.headers.update(
+            {
+                "X-PrefabCloud-Client-Version": f"prefab-cloud-python-{getattr(options, 'version', 'development')}"
+            }
+        )
         # Initialize a cache (here with a maximum of 2 entries).
         self.cache = LRUCache(max_size=2)
 
@@ -189,12 +194,15 @@ def _update_cache(self, url: str, response: Response) -> None:
             max_age = int(m.group(1))
         expires_at = time.time() + max_age if max_age > 0 else 0
         if (etag is not None or max_age > 0) and expires_at > time.time():
-            self.cache.set(url, CacheEntry(
-                data=response.content,
-                etag=etag,
-                expires_at=expires_at,
-                url=response.url
-            ))
+            self.cache.set(
+                url,
+                CacheEntry(
+                    data=response.content,
+                    etag=etag,
+                    expires_at=expires_at,
+                    url=response.url,
+                ),
+            )
             response.headers["X-Cache"] = "MISS"
 
     def _send_request(self, method: str, url: str, **kwargs) -> Response:
@@ -208,7 +216,14 @@ def _send_request(self, method: str, url: str, **kwargs) -> Response:
         wait=wait_exponential(multiplier=1, min=0.05, max=2),
         retry=retry_if_exception_type((RequestException, ConnectionError, OSError)),
     )
-    def resilient_request(self, path, method="GET", allow_cache: bool = False, hosts: list[str] = None, **kwargs) -> Response:
+    def resilient_request(
+        self,
+        path,
+        method="GET",
+        allow_cache: bool = False,
+        hosts: list[str] = None,
+        **kwargs,
+    ) -> Response:
         """
         Makes a resilient (retrying) request.
 
@@ -237,4 +252,4 @@ def resilient_request(self, path, method="GET", allow_cache: bool = False, hosts
                     resp.url = cached.url
                     return resp
             self._update_cache(url, response)
-        return response
+        return response

prefab_cloud_python/config_client.py

@@ -157,8 +157,12 @@ def load_initial_data(self):
 
     def load_checkpoint_from_api_cdn(self):
         try:
+            hwm = self.config_loader.highwater_mark
             response = self.api_client.resilient_request(
-                "/api/v1/configs/0", auth=("authuser", self.options.api_key), timeout=4, allow_cache=True
+                "/api/v1/configs/" + str(hwm),
+                auth=("authuser", self.options.api_key),
+                timeout=4,
+                allow_cache=True,
             )
             if response.ok:
                 configs = Prefab.Configs.FromString(response.content)

tests/test_api_client.py

@@ -3,11 +3,13 @@
 from requests import Response
 from prefab_cloud_python._requests import ApiClient
 
+
 # Dummy options for testing.
 class DummyOptions:
     prefab_api_urls = ["https://a.example.com", "https://b.example.com"]
     version = "1.0"
 
+
 class TestApiClient(unittest.TestCase):
     def setUp(self):
         self.options = DummyOptions()
@@ -16,7 +18,13 @@ def setUp(self):
         # patch _get_attempt_number to always return 1.
         self.client._get_attempt_number = lambda: 1
 
-    def create_response(self, status_code=200, content=b"dummy", headers=None, url="https://a.example.com/api/v1/configs/0"):
+    def create_response(
+        self,
+        status_code=200,
+        content=b"dummy",
+        headers=None,
+        url="https://a.example.com/api/v1/configs/0",
+    ):
         resp = Response()
         resp.status_code = status_code
         resp._content = content
@@ -31,7 +39,7 @@ def test_no_cache(self, mock_send_request):
             status_code=200,
             content=b"response_no_cache",
             headers={"Cache-Control": "max-age=60", "ETag": "abc"},
-            url="https://a.example.com/api/v1/configs/0"
+            url="https://a.example.com/api/v1/configs/0",
         )
         mock_send_request.return_value = response
 
@@ -47,7 +55,7 @@ def test_cache_miss_and_hit(self, mock_send_request):
             status_code=200,
             content=b"cached_response",
             headers={"Cache-Control": "max-age=60", "ETag": "abc"},
-            url="https://a.example.com/api/v1/configs/0"
+            url="https://a.example.com/api/v1/configs/0",
         )
         mock_send_request.return_value = response
 
@@ -61,7 +69,7 @@ def test_cache_miss_and_hit(self, mock_send_request):
             status_code=200,
             content=b"new_response",
             headers={"Cache-Control": "max-age=60", "ETag": "def"},
-            url="https://a.example.com/api/v1/configs/0"
+            url="https://a.example.com/api/v1/configs/0",
         )
         mock_send_request.return_value = new_response
 
@@ -78,7 +86,7 @@ def test_304_returns_cached_response(self, mock_send_request):
             status_code=200,
             content=b"cached_response",
             headers={"Cache-Control": "max-age=60", "ETag": "abc"},
-            url="https://a.example.com/api/v1/configs/0"
+            url="https://a.example.com/api/v1/configs/0",
         )
         mock_send_request.return_value = response
         resp1 = self.client.resilient_request("/api/v1/configs/0", allow_cache=True)
@@ -91,13 +99,14 @@ def test_304_returns_cached_response(self, mock_send_request):
             status_code=304,
             content=b"",
             headers={},
-            url="https://a.example.com/api/v1/configs/0"
+            url="https://a.example.com/api/v1/configs/0",
         )
         mock_send_request.return_value = response_304
         resp2 = self.client.resilient_request("/api/v1/configs/0", allow_cache=True)
         self.assertEqual(resp2.status_code, 200)
         self.assertEqual(resp2.content, b"cached_response")
         self.assertEqual(resp2.headers.get("X-Cache"), "HIT")
 
+
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()