polishing

Author: chillaq

splitio/api/__init__.py

@@ -14,6 +14,13 @@ def status_code(self):
         """Return HTTP status code."""
         return self._status_code
 
+class APIUriException(APIException):
+    """Exception to raise when an API call fails due to 414  http error."""
+
+    def __init__(self, custom_message, status_code=None):
+        """Constructor."""
+        APIException.__init__(self, custom_message)
+
 def headers_from_metadata(sdk_metadata, client_key=None):
     """
     Generate a dict with headers required by data-recording API endpoints.

splitio/sync/manager.py

@@ -285,7 +285,7 @@ def __init__(self, synchronizer):  # pylint:disable=too-many-arguments
         :param synchronizer: synchronizers for performing start/stop logic
         :type synchronizer: splitio.sync.synchronizer.Synchronizer
         """
-        super().__init__(synchronizer)
+        RedisManagerBase.__init__(self, synchronizer)
 
     def stop(self, blocking):
         """
@@ -308,7 +308,7 @@ def __init__(self, synchronizer):  # pylint:disable=too-many-arguments
         :param synchronizer: synchronizers for performing start/stop logic
         :type synchronizer: splitio.sync.synchronizer.Synchronizer
         """
-        super().__init__(synchronizer)
+        RedisManagerBase.__init__(self, synchronizer)
 
     async def stop(self, blocking):
         """

splitio/sync/split.py

@@ -8,7 +8,7 @@
 import hashlib
 from enum import Enum
 
-from splitio.api import APIException
+from splitio.api import APIException, APIUriException
 from splitio.api.commons import FetchOptions
 from splitio.client.input_validator import validate_flag_sets
 from splitio.models import splits
@@ -77,7 +77,7 @@ def __init__(self, feature_flag_api, feature_flag_storage):
         :param feature_flag_storage: Feature Flag Storage.
         :type feature_flag_storage: splitio.storage.InMemorySplitStorage
         """
-        super().__init__(feature_flag_api, feature_flag_storage)
+        SplitSynchronizerBase.__init__(self, feature_flag_api, feature_flag_storage)
 
     def _fetch_until(self, fetch_options, till=None):
         """
@@ -104,12 +104,16 @@ def _fetch_until(self, fetch_options, till=None):
             try:
                 feature_flag_changes = self._api.fetch_splits(change_number, fetch_options)
             except APIException as exc:
+                if exc._status_code is not None and exc._status_code == 414:
+                    _LOGGER.error('SDK Initialization: the amount of flag sets provided are big causing uri length error.')
+                    _LOGGER.debug('Exception information: ', exc_info=True)
+                    raise APIUriException("URI is too long due to FlagSets count")
+
                 _LOGGER.error('Exception raised while fetching feature flags')
                 _LOGGER.debug('Exception information: ', exc_info=True)
                 raise exc
 
-            fetched_feature_flags = []
-            [fetched_feature_flags.append(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('splits', [])]
+            fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('splits', [])]
             segment_list = update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes['till'])
             if feature_flag_changes['till'] == feature_flag_changes['since']:
                 return feature_flag_changes['till'], segment_list
@@ -195,7 +199,7 @@ def __init__(self, feature_flag_api, feature_flag_storage):
         :param feature_flag_storage: Feature Flag Storage.
         :type feature_flag_storage: splitio.storage.InMemorySplitStorage
         """
-        super().__init__(feature_flag_api, feature_flag_storage)
+        SplitSynchronizerBase.__init__(self, feature_flag_api, feature_flag_storage)
 
     async def _fetch_until(self, fetch_options, till=None):
         """
@@ -222,12 +226,16 @@ async def _fetch_until(self, fetch_options, till=None):
             try:
                 feature_flag_changes = await self._api.fetch_splits(change_number, fetch_options)
             except APIException as exc:
+                if exc._status_code is not None and exc._status_code == 414:
+                    _LOGGER.error('Exception caught: the amount of flag sets provided are big causing uri length error.')
+                    _LOGGER.debug('Exception information: ', exc_info=True)
+                    raise APIUriException("URI is too long due to FlagSets count")
+
                 _LOGGER.error('Exception raised while fetching feature flags')
                 _LOGGER.debug('Exception information: ', exc_info=True)
                 raise exc
 
-            fetched_feature_flags = []
-            [fetched_feature_flags.append(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('splits', [])]
+            fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('splits', [])]
             segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes['till'])
             if feature_flag_changes['till'] == feature_flag_changes['since']:
                 return feature_flag_changes['till'], segment_list
@@ -597,7 +605,7 @@ def synchronize_splits(self, till=None):  # pylint:disable=unused-argument
         try:
             return self._synchronize_json() if self._localhost_mode == LocalhostMode.JSON else self._synchronize_legacy()
         except Exception as exc:
-            _LOGGER.error(str(exc))
+            _LOGGER.debug('Exception: ', exc_info=True)
             raise APIException("Error fetching feature flags information") from exc
 
     def _synchronize_legacy(self):
@@ -639,7 +647,7 @@ def _synchronize_json(self):
             segment_list = update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, till)
             return segment_list
         except Exception as exc:
-            _LOGGER.debug(exc)
+            _LOGGER.debug('Exception: ', exc_info=True)
             raise ValueError("Error reading feature flags from json.") from exc
 
     def _read_feature_flags_from_json_file(self, filename):
@@ -658,7 +666,7 @@ def _read_feature_flags_from_json_file(self, filename):
             santitized = self._sanitize_feature_flag(parsed)
             return santitized['splits'], santitized['till']
         except Exception as exc:
-            _LOGGER.error(str(exc))
+            _LOGGER.debug('Exception: ', exc_info=True)
             raise ValueError("Error parsing file %s. Make sure it's readable." % filename) from exc
 
 
@@ -741,7 +749,7 @@ async def synchronize_splits(self, till=None):  # pylint:disable=unused-argument
         try:
             return await self._synchronize_json() if self._localhost_mode == LocalhostMode.JSON else await self._synchronize_legacy()
         except Exception as exc:
-            _LOGGER.error(str(exc))
+            _LOGGER.debug('Exception: ', exc_info=True)
             raise APIException("Error fetching feature flags information") from exc
 
     async def _synchronize_legacy(self):
@@ -783,7 +791,7 @@ async def _synchronize_json(self):
             segment_list = await update_feature_flag_storage_async(self._feature_flag_storage, fetched_feature_flags, till)
             return segment_list
         except Exception as exc:
-            _LOGGER.debug(exc)
+            _LOGGER.debug('Exception: ', exc_info=True)
             raise ValueError("Error reading feature flags from json.") from exc
 
     async def _read_feature_flags_from_json_file(self, filename):
@@ -802,5 +810,5 @@ async def _read_feature_flags_from_json_file(self, filename):
             santitized = self._sanitize_feature_flag(parsed)
             return santitized['splits'], santitized['till']
         except Exception as exc:
-            _LOGGER.error(str(exc))
+            _LOGGER.debug('Exception: ', exc_info=True)
             raise ValueError("Error parsing file %s. Make sure it's readable." % filename) from exc

splitio/sync/synchronizer.py

@@ -6,7 +6,7 @@
 import time
 
 from splitio.optional.loaders import asyncio
-from splitio.api import APIException
+from splitio.api import APIException, APIUriException
 from splitio.util.backoff import Backoff
 from splitio.sync.split import _ON_DEMAND_FETCH_BACKOFF_BASE, _ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES, _ON_DEMAND_FETCH_BACKOFF_MAX_WAIT, LocalhostMode
 
@@ -252,7 +252,6 @@ def __init__(self, split_synchronizers, split_tasks):
             self._periodic_data_recording_tasks.append(self._split_tasks.unique_keys_task)
         if self._split_tasks.clear_filter_task:
             self._periodic_data_recording_tasks.append(self._split_tasks.clear_filter_task)
-        self._break_sync_all = False
 
     @property
     def split_sync(self):
@@ -354,7 +353,7 @@ def __init__(self, split_synchronizers, split_tasks):
         :param split_tasks: tasks for starting/stopping tasks
         :type split_tasks: splitio.sync.synchronizer.SplitTasks
         """
-        super().__init__(split_synchronizers, split_tasks)
+        SynchronizerInMemoryBase.__init__(self, split_synchronizers, split_tasks)
 
     def _synchronize_segments(self):
         _LOGGER.debug('Starting segments synchronization')
@@ -385,7 +384,6 @@ def synchronize_splits(self, till, sync_segments=True):
         :returns: whether the synchronization was successful or not.
         :rtype: bool
         """
-        self._break_sync_all = False
         _LOGGER.debug('Starting splits synchronization')
         try:
             new_segments = []
@@ -401,9 +399,12 @@ def synchronize_splits(self, till, sync_segments=True):
                 else:
                     _LOGGER.debug('Segment sync scheduled.')
             return True
+        except APIUriException as exc:
+            _LOGGER.error('Failed syncing feature flags due to long URI')
+            _LOGGER.debug('Error: ', exc_info=True)
+            return False
+
         except APIException as exc:
-            if exc._status_code is not None and exc._status_code == 414:
-                self._break_sync_all = True
             _LOGGER.error('Failed syncing feature flags')
             _LOGGER.debug('Error: ', exc_info=True)
             return False
@@ -428,12 +429,16 @@ def sync_all(self, max_retry_attempts=_SYNC_ALL_NO_RETRIES):
 
                 # All is good
                 return
+            except APIUriException as exc:
+                _LOGGER.error("URI too long exception, aborting retries.")
+                _LOGGER.debug('Error: ', exc_info=True)
+                break
             except Exception as exc:  # pylint:disable=broad-except
                 _LOGGER.error("Exception caught when trying to sync all data: %s", str(exc))
                 _LOGGER.debug('Error: ', exc_info=True)
                 if max_retry_attempts != _SYNC_ALL_NO_RETRIES:
                     retry_attempts += 1
-                    if retry_attempts > max_retry_attempts or self._break_sync_all:
+                    if retry_attempts > max_retry_attempts:
                         break
                 how_long = self._backoff.get()
                 time.sleep(how_long)
@@ -508,7 +513,7 @@ def __init__(self, split_synchronizers, split_tasks):
         :param split_tasks: tasks for starting/stopping tasks
         :type split_tasks: splitio.sync.synchronizer.SplitTasks
         """
-        super().__init__(split_synchronizers, split_tasks)
+        SynchronizerInMemoryBase.__init__(self, split_synchronizers, split_tasks)
         self.stop_periodic_data_recording_task = None
 
     async def _synchronize_segments(self):
@@ -540,7 +545,6 @@ async def synchronize_splits(self, till, sync_segments=True):
         :returns: whether the synchronization was successful or not.
         :rtype: bool
         """
-        self._break_sync_all = False
         _LOGGER.debug('Starting feature flags synchronization')
         try:
             new_segments = []
@@ -556,9 +560,12 @@ async def synchronize_splits(self, till, sync_segments=True):
                 else:
                     _LOGGER.debug('Segment sync scheduled.')
             return True
+        except APIUriException as exc:
+            _LOGGER.error('Failed syncing feature flags due to long URI')
+            _LOGGER.debug('Error: ', exc_info=True)
+            return False
+
         except APIException as exc:
-            if exc._status_code is not None and exc._status_code == 414:
-                self._break_sync_all = True
             _LOGGER.error('Failed syncing feature flags')
             _LOGGER.debug('Error: ', exc_info=True)
             return False
@@ -583,12 +590,16 @@ async def sync_all(self, max_retry_attempts=_SYNC_ALL_NO_RETRIES):
 
                 # All is good
                 return
+            except APIUriException as exc:
+                _LOGGER.error("URI too long exception, aborting retries.")
+                _LOGGER.debug('Error: ', exc_info=True)
+                break
             except Exception as exc:  # pylint:disable=broad-except
                 _LOGGER.error("Exception caught when trying to sync all data: %s", str(exc))
                 _LOGGER.debug('Error: ', exc_info=True)
                 if max_retry_attempts != _SYNC_ALL_NO_RETRIES:
                     retry_attempts += 1
-                    if retry_attempts > max_retry_attempts or self._break_sync_all:
+                    if retry_attempts > max_retry_attempts:
                         break
                 how_long = self._backoff.get()
                 time.sleep(how_long)
@@ -734,7 +745,7 @@ def __init__(self, split_synchronizers, split_tasks):
         :param split_tasks: tasks for starting/stopping tasks
         :type split_tasks: splitio.sync.synchronizer.SplitTasks
         """
-        super().__init__(split_synchronizers, split_tasks)
+        RedisSynchronizerBase.__init__(self, split_synchronizers, split_tasks)
 
     def shutdown(self, blocking):
         """
@@ -779,7 +790,7 @@ def __init__(self, split_synchronizers, split_tasks):
         :param split_tasks: tasks for starting/stopping tasks
         :type split_tasks: splitio.sync.synchronizer.SplitTasks
         """
-        super().__init__(split_synchronizers, split_tasks)
+        RedisSynchronizerBase.__init__(self, split_synchronizers, split_tasks)
         self.stop_periodic_data_recording_task = None
 
     async def shutdown(self, blocking):
@@ -895,7 +906,7 @@ def __init__(self, split_synchronizers, split_tasks, localhost_mode):
         :param split_tasks: tasks for starting/stopping tasks
         :type split_tasks: splitio.sync.synchronizer.SplitTasks
         """
-        super().__init__(split_synchronizers, split_tasks, localhost_mode)
+        LocalhostSynchronizerBase.__init__(self, split_synchronizers, split_tasks, localhost_mode)
 
     def sync_all(self, till=None):
         """
@@ -969,7 +980,7 @@ def __init__(self, split_synchronizers, split_tasks, localhost_mode):
         :param split_tasks: tasks for starting/stopping tasks
         :type split_tasks: splitio.sync.synchronizer.SplitTasks
         """
-        super().__init__(split_synchronizers, split_tasks, localhost_mode)
+        LocalhostSynchronizerBase.__init__(self, split_synchronizers, split_tasks, localhost_mode)
 
     async def sync_all(self, till=None):
         """