Merge pull request #265 from splitio/BloomFilter

BloomFilter implementation

Author: chillaq

setup.py

@@ -19,6 +19,7 @@
     'pyyaml>=5.4',
     'docopt>=0.6.2',
     'enum34;python_version<"3.4"',
+    'bloom-filter2>=2.0.0',
 ]
 
 with open(path.join(path.abspath(path.dirname(__file__)), 'splitio', 'version.py')) as f:

splitio/engine/filters/__init__.py

@@ -0,0 +1,28 @@
+import abc
+
+class BaseFilter(object, metaclass=abc.ABCMeta):
+    """Impressions Filter interface."""
+
+    @abc.abstractmethod
+    def add(self, data):
+        """
+        Return a boolean flag
+
+        """
+        pass
+
+    @abc.abstractmethod
+    def contains(self, data):
+        """
+        Return a boolean flag
+
+        """
+        pass
+
+    @abc.abstractmethod
+    def clear(self):
+        """
+        No return
+
+        """
+        pass

splitio/engine/filters/bloom_filter.py

@@ -0,0 +1,52 @@
+from splitio.engine.filters import BaseFilter
+from bloom_filter2 import BloomFilter as BloomFilter2
+
+class BloomFilter(BaseFilter):
+    """Optimized mode strategy."""
+
+    def __init__(self, max_elements=5000, error_rate=0.01):
+        """
+        Construct a bloom filter instance.
+
+        :param max_element: maximum elements in the filter
+        :type string:
+
+        :param error_rate: error rate for the false positives, reduce it will consume more memory
+        :type numeric:
+        """
+        self._max_elements = max_elements
+        self._error_rate = error_rate
+        self._imps_bloom_filter = BloomFilter2(max_elements=self._max_elements, error_rate=self._error_rate)
+
+    def add(self, data):
+        """
+        Add an item to the bloom filter instance.
+
+        :param data: element to be added
+        :type string:
+
+        :return: True if successful
+        :rtype: boolean
+        """
+        self._imps_bloom_filter.add(data)
+        return data in self._imps_bloom_filter
+
+    def contains(self, data):
+        """
+        Check if an item exist in the bloom filter instance.
+
+        :param data: element to be checked
+        :type string:
+
+        :return: True if exist
+        :rtype: boolean
+        """
+        return data in self._imps_bloom_filter
+
+    def clear(self):
+        """
+        Destroy the current filter instance and create new one.
+
+        """
+        self._imps_bloom_filter.close()
+        self._imps_bloom_filter = BloomFilter2(max_elements=self._max_elements, error_rate=self._error_rate)

tests/engine/test_bloom_filter.py

@@ -0,0 +1,55 @@
+"""BloomFilter unit tests."""
+
+from random import random
+import uuid
+from splitio.engine.filters.bloom_filter import BloomFilter
+
+class BloomFilterTests(object):
+    """StandardRecorderTests test cases."""
+
+    def test_bloom_filter_methods(self, mocker):
+        bloom_filter = BloomFilter()
+        key1 = str(uuid.uuid4())
+        key2 = str(uuid.uuid4())
+        bloom_filter.add(key1)
+
+        assert(bloom_filter.contains(key1))
+        assert(not bloom_filter.contains(key2))
+
+        bloom_filter.clear()
+        assert(not bloom_filter.contains(key1))
+
+        bloom_filter.add(key1)
+        bloom_filter.add(key2)
+        assert(bloom_filter.contains(key1))
+        assert(bloom_filter.contains(key2))
+
+    def test_bloom_filter_error_percentage(self, mocker):
+        arr_storage = []
+        total_sample = 20000
+        error_rate = 0.01
+        bloom_filter = BloomFilter(total_sample, error_rate)
+
+        for x in range(1, total_sample):
+            myuuid = str(uuid.uuid4())
+            bloom_filter.add(myuuid)
+            arr_storage.append(myuuid)
+
+        false_positive_count = 0
+        for x in range(1, total_sample):
+            y = int(random()*total_sample*5)
+            if y > total_sample - 2:
+                myuuid = str(uuid.uuid4())
+                if myuuid in arr_storage:
+                    # False Negative
+                    assert(bloom_filter.contains(myuuid))
+                else:
+                    if bloom_filter.contains(myuuid):
+                        # False Positive
+                        false_positive_count = false_positive_count + 1
+            else:
+                myuuid = arr_storage[y]
+                assert(bloom_filter.contains(myuuid))
+                # False Negative
+
+        assert(false_positive_count/total_sample <= error_rate)