fix remaining tests

mredolatti · mredolatti · commit 19fc89ccb5e4 · 2020-10-08T12:13:20.000-03:00
diff --git a/splitio/api/impressions.py b/splitio/api/impressions.py
@@ -7,12 +7,13 @@
 
 from splitio.api import APIException, headers_from_metadata
 from splitio.api.client import HttpClientException
+from splitio.engine.impressions import ImpressionsMode
 
 
 class ImpressionsAPI(object):  # pylint: disable=too-few-public-methods
     """Class that uses an httpClient to communicate with the impressions API."""
 
-    def __init__(self, client, apikey, sdk_metadata):
+    def __init__(self, client, apikey, sdk_metadata, mode=ImpressionsMode.OPTIMIZED):
         """
         Class constructor.
 
@@ -25,6 +26,7 @@ def __init__(self, client, apikey, sdk_metadata):
         self._client = client
         self._apikey = apikey
         self._metadata = headers_from_metadata(sdk_metadata)
+        self._metadata['SplitSDKImpressionsMode'] = mode.name
 
     @staticmethod
     def _build_bulk(impressions):
@@ -47,7 +49,8 @@ def _build_bulk(impressions):
                         'm': impression.time,
                         'c': impression.change_number,
                         'r': impression.label,
-                        'b': impression.bucketing_key
+                        'b': impression.bucketing_key,
+                        'pt': impression.previous_time
                     }
                     for impression in imps
                 ]
@@ -74,7 +77,7 @@ def _build_counters(counters):
                 {
                     'f': pf_count.feature,
                     'm': pf_count.timeframe,
-                    'c': pf_count.count
+                    'rc': pf_count.count
                 } for pf_count in counters
             ]
         }
diff --git a/splitio/client/client.py b/splitio/client/client.py
@@ -11,6 +11,7 @@
 from splitio.models.events import Event, EventWrapper
 from splitio.models.telemetry import get_latency_bucket_index
 from splitio.client import input_validator
+from splitio.util import utctime_ms
 
 
 class Client(object):  # pylint: disable=too-many-instance-attributes
@@ -115,7 +116,7 @@ def _make_evaluation(self, key, feature, attributes, method_name, metric_name):
                 result['impression']['label'],
                 result['impression']['change_number'],
                 bucketing_key,
-                start
+                utctime_ms(),
             )
 
             self._record_stats([(impression, attributes)], start, metric_name)
@@ -131,7 +132,7 @@ def _make_evaluation(self, key, feature, attributes, method_name, metric_name):
                     Label.EXCEPTION,
                     self._split_storage.get_change_number(),
                     bucketing_key,
-                    start
+                    utctime_ms(),
                 )
                 self._record_stats([(impression, attributes)], start, metric_name)
             except Exception:  # pylint: disable=broad-except
@@ -178,7 +179,7 @@ def _make_evaluations(self, key, features, attributes, method_name, metric_name)
                                                         result['impression']['label'],
                                                         result['impression']['change_number'],
                                                         bucketing_key,
-                                                        start)
+                                                        utctime_ms())
 
                     bulk_impressions.append(impression)
                     treatments[feature] = (result['treatment'], result['configurations'])
@@ -389,7 +390,7 @@ def track(self, key, traffic_type, event_type, value=None, properties=None):
             traffic_type_name=traffic_type,
             event_type_id=event_type,
             value=value,
-            timestamp=int(time.time()*1000),
+            timestamp=utctime_ms(),
             properties=properties,
         )
         return self._events_storage.put([EventWrapper(
diff --git a/splitio/client/factory.py b/splitio/client/factory.py
@@ -74,7 +74,7 @@ def __init__(  # pylint: disable=too-many-arguments
             apikey,
             storages,
             labels_enabled,
-            impressions_manager=None,
+            impressions_manager,
             apis=None,
             tasks=None,
             sdk_ready_flag=None,
@@ -139,7 +139,7 @@ def client(self):
         This client is only a set of references to structures hold by the factory.
         Creating one a fast operation and safe to be used anywhere.
         """
-        return Client(self, self._labels_enabled, self._impressions_manager)
+        return Client(self, self._impressions_manager, self._labels_enabled)
 
     def manager(self):
         """
@@ -249,7 +249,7 @@ def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None):  # py
     apis = {
         'splits': SplitsAPI(http_client, api_key),
         'segments': SegmentsAPI(http_client, api_key),
-        'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata),
+        'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']),
         'events': EventsAPI(http_client, api_key, sdk_metadata),
         'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
     }
diff --git a/splitio/engine/impressions.py b/splitio/engine/impressions.py
@@ -65,11 +65,11 @@ def _stringify(self, impression):
         :returns: a string representation of the impression
         :rtype: str
         """
-        return self._PATTERN % (impression.matching_key,
-                                impression.feature_name,
-                                impression.treatment,
-                                impression.label,
-                                impression.change_number)
+        return self._PATTERN % (impression.matching_key if impression.matching_key else 'UNKNOWN',
+                                impression.feature_name if impression.feature_name else 'UNKNOWN',
+                                impression.treatment if impression.treatment else 'UNKNOWN',
+                                impression.label if impression.label else 'UNKNOWN',
+                                impression.change_number if impression.change_number else 0)
 
     def process(self, impression):
         """
diff --git a/splitio/tasks/events_sync.py b/splitio/tasks/events_sync.py
@@ -70,12 +70,9 @@ def _send_events(self):
 
         try:
             self._events_api.flush_events(to_send)
-        except APIException as exc:
-            self._logger.error(
-                'Exception raised while reporting events: %s -- %d',
-                exc.message,
-                exc.status_code
-            )
+        except APIException:
+            self._logger.error('Exception raised while reporting events')
+            self._logger.debug('Exception information: ', exc_info=True)
             self._add_to_failed_queue(to_send)
 
     def start(self):
diff --git a/splitio/tasks/impressions_sync.py b/splitio/tasks/impressions_sync.py
@@ -71,12 +71,9 @@ def _send_impressions(self):
 
         try:
             self._impressions_api.flush_impressions(to_send)
-        except APIException as exc:
-            self._logger.error(
-                'Exception raised while reporting impressions: %s -- %d',
-                exc.message,
-                exc.status_code
-            )
+        except APIException:
+            self._logger.error('Exception raised while reporting impressions')
+            self._logger.debug('Exception information: ', exc_info=True)
             self._add_to_failed_queue(to_send)
 
     def start(self):
@@ -104,7 +101,7 @@ def flush(self):
 class ImpressionsCountSyncTask(BaseSynchronizationTask):
     """Impressions synchronization task uses an asynctask.AsyncTask to send impressions."""
 
-    _PERIOD = 30 * 60 # 30 minutes
+    _PERIOD = 5 # 30 * 60 # 30 minutes
 
     def __init__(self, impressions_api, impressions_manager):
         """
@@ -124,14 +121,14 @@ def __init__(self, impressions_api, impressions_manager):
     def _send_counters(self):
         """Send impressions from both the failed and new queues."""
         to_send = self._impressions_manager.get_counts()
+        if not to_send:
+            return
+
         try:
             self._impressions_api.flush_counters(to_send)
-        except APIException as exc:
-            self._logger.error(
-                'Exception raised while reporting impressions: %s -- %d',
-                exc.message,
-                exc.status_code
-            )
+        except APIException:
+            self._logger.error('Exception raised while reporting impression counts')
+            self._logger.debug('Exception information: ', exc_info=True)
 
     def start(self):
         """Start executing the impressions synchronization task."""
diff --git a/splitio/tasks/segment_sync.py b/splitio/tasks/segment_sync.py
@@ -46,7 +46,8 @@ def _update_segment(self, segment_name):
         try:
             segment_changes = self._segment_api.fetch_segment(segment_name, since)
         except APIException:
-            self._logger.error('Error fetching segments')
+            self._logger.error('Exception raised while fetching segment %s', segment_name)
+            self._logger.debug('Exception information: ', exc_info=True)
             return False
 
         if since == -1:  # first time fetching the segment
diff --git a/splitio/tasks/split_sync.py b/splitio/tasks/split_sync.py
@@ -42,7 +42,8 @@ def _update_splits(self):
         try:
             split_changes = self._api.fetch_splits(till)
         except APIException:
-            self._logger.error('Failed to fetch split from servers')
+            self._logger.error('Exception raised while fetching splits')
+            self._logger.debug('Exception information: ', exc_info=True)
             return False
 
         for split in split_changes.get('splits', []):
diff --git a/splitio/tasks/telemetry_sync.py b/splitio/tasks/telemetry_sync.py
@@ -37,20 +37,23 @@ def _flush_telemetry(self):
                 self._api.flush_latencies(latencies)
         except APIException:
             self._logger.error('Failed send telemetry/latencies to split BE.')
+            self._logger.debug('Exception information: ', exc_info=True)
 
         try:
             counters = self._storage.pop_counters()
             if counters:
                 self._api.flush_counters(counters)
         except APIException:
             self._logger.error('Failed send telemetry/counters to split BE.')
+            self._logger.debug('Exception information: ', exc_info=True)
 
         try:
             gauges = self._storage.pop_gauges()
             if gauges:
                 self._api.flush_gauges(gauges)
         except APIException:
             self._logger.error('Failed send telemetry/gauges to split BE.')
+            self._logger.debug('Exception information: ', exc_info=True)
 
     def start(self):
         """Start the task."""
diff --git a/splitio/tasks/util/asynctask.py b/splitio/tasks/util/asynctask.py
@@ -102,11 +102,13 @@ def _execution_wrapper(self):
                 except queue.Empty:
                     # If no message was received, the timeout has expired
                     # and we're ready for a new execution
-                    if not _safe_run(self._main):
-                        _LOGGER.error(
-                            "An error occurred when executing the task. "
-                            "Retrying after perio expires"
-                        )
+                    pass
+
+                if not _safe_run(self._main):
+                    _LOGGER.error(
+                        "An error occurred when executing the task. "
+                        "Retrying after perio expires"
+                    )
         finally:
             self._cleanup()
 
diff --git a/splitio/tasks/uwsgi_wrappers.py b/splitio/tasks/uwsgi_wrappers.py
@@ -3,7 +3,7 @@
 import logging
 import time
 
-from splitio.client.config import DEFAULT_CONFIG
+from splitio.client.config import sanitize as sanitize_config
 from splitio.client.util import get_metadata
 from splitio.storage.adapters.uwsgi_cache import get_uwsgi
 from splitio.storage.uwsgi import UWSGIEventStorage, UWSGIImpressionStorage, \
@@ -34,9 +34,7 @@ def _get_config(user_config):
     :return: Calculated configuration.
     :rtype: dict
     """
-    sdk_config = DEFAULT_CONFIG.copy()
-    sdk_config.update(user_config)
-    return sdk_config
+    return sanitize_config(user_config['apikey'], user_config)
 
 
 def uwsgi_update_splits(user_config):
@@ -113,7 +111,8 @@ def uwsgi_report_impressions(user_config):
         ImpressionsAPI(
             HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
             config['apikey'],
-            metadata
+            metadata,
+            config['impressionsMode']
         ),
         storage,
         None, # Period not needed. Task is being triggered manually.
diff --git a/tests/api/test_impressions_api.py b/tests/api/test_impressions_api.py
@@ -3,7 +3,7 @@
 import pytest
 from splitio.api import impressions, client, APIException
 from splitio.models.impressions import Impression
-from splitio.engine.impressions import Counter
+from splitio.engine.impressions import Counter, ImpressionsMode
 from splitio.client.util import get_metadata
 from splitio.client.config import DEFAULT_CONFIG
 from splitio.version import __version__
@@ -19,13 +19,13 @@ class ImpressionsAPITests(object):
     expectedImpressions = [{
         'f': 'f1',
         'i': [
-            {'k': 'k1', 'b': 'b1', 't': 'on', 'r': 'l1', 'm': 321654, 'c': 123456},
-            {'k': 'k3', 'b': 'b1', 't': 'on', 'r': 'l1', 'm': 321654, 'c': 123456},
+            {'k': 'k1', 'b': 'b1', 't': 'on', 'r': 'l1', 'm': 321654, 'c': 123456, 'pt': None},
+            {'k': 'k3', 'b': 'b1', 't': 'on', 'r': 'l1', 'm': 321654, 'c': 123456, 'pt': None},
         ],
     }, {
         'f': 'f2',
         'i': [
-            {'k': 'k2', 'b': 'b1', 't': 'off', 'r': 'l1', 'm': 321654, 'c': 123456},
+            {'k': 'k2', 'b': 'b1', 't': 'off', 'r': 'l1', 'm': 321654, 'c': 123456, 'pt': None},
         ]
     }]
 
@@ -38,10 +38,10 @@ class ImpressionsAPITests(object):
 
     expected_counters = {
         'pf': [
-            {'f': 'f1', 'm': 123, 'c': 2},
-            {'f': 'f2', 'm': 123, 'c': 123},
-            {'f': 'f1', 'm': 456, 'c': 111},
-            {'f': 'f2', 'm': 456, 'c': 222},
+            {'f': 'f1', 'm': 123, 'rc': 2},
+            {'f': 'f2', 'm': 123, 'rc': 123},
+            {'f': 'f1', 'm': 456, 'rc': 111},
+            {'f': 'f2', 'm': 456, 'rc': 222},
         ]
     }
 
@@ -64,7 +64,8 @@ def test_post_impressions(self, mocker):
         assert call_made[2]['extra_headers'] == {
             'SplitSDKVersion': 'python-%s' % __version__,
             'SplitSDKMachineIP': '123.123.123.123',
-            'SplitSDKMachineName': 'some_machine_name'
+            'SplitSDKMachineName': 'some_machine_name',
+            'SplitSDKImpressionsMode': 'OPTIMIZED'
         }
 
         # validate key-value args (body)
@@ -86,7 +87,7 @@ def test_post_impressions_ip_address_disabled(self, mocker):
         cfg = DEFAULT_CONFIG.copy()
         cfg.update({'IPAddressesEnabled': False})
         sdk_metadata = get_metadata(cfg)
-        impressions_api = impressions.ImpressionsAPI(httpclient, 'some_api_key', sdk_metadata)
+        impressions_api = impressions.ImpressionsAPI(httpclient, 'some_api_key', sdk_metadata, ImpressionsMode.DEBUG)
         response = impressions_api.flush_impressions(self.impressions)
 
         call_made = httpclient.post.mock_calls[0]
@@ -97,6 +98,7 @@ def test_post_impressions_ip_address_disabled(self, mocker):
         # validate key-value args (headers)
         assert call_made[2]['extra_headers'] == {
             'SplitSDKVersion': 'python-%s' % __version__,
+            'SplitSDKImpressionsMode': 'DEBUG'
         }
 
         # validate key-value args (body)
@@ -121,7 +123,8 @@ def test_post_counters(self, mocker):
         assert call_made[2]['extra_headers'] == {
             'SplitSDKVersion': 'python-%s' % __version__,
             'SplitSDKMachineIP': '123.123.123.123',
-            'SplitSDKMachineName': 'some_machine_name'
+            'SplitSDKMachineName': 'some_machine_name',
+            'SplitSDKImpressionsMode': 'OPTIMIZED'
         }
 
         # validate key-value args (body)
diff --git a/tests/client/test_client.py b/tests/client/test_client.py
diff --git a/tests/client/test_factory.py b/tests/client/test_factory.py
diff --git a/tests/integration/test_client_e2e.py b/tests/integration/test_client_e2e.py
