修复bloomfiler 扩容bug

Boris-code · Jun 7, 2022 · c5d2a68 · c5d2a68
1 parent e1b47a9
commit c5d2a68
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 18 deletions.
diff --git a/feapder/dedup/bitarray.py b/feapder/dedup/bitarray.py
@@ -138,6 +138,6 @@ def count(self, value=True):
         if count:
             return int(count)
         else:
-            count = self.redis_db.bitcount(self.name)
+            count = self.redis_db.bitcount(self.name)  # 被设置为 1 的比特位的数量
             self.redis_db.strset(self.count_cached_name, count, ex=1800)  # 半小时过期
             return count
diff --git a/feapder/dedup/bloomfilter.py b/feapder/dedup/bloomfilter.py
@@ -145,24 +145,18 @@ def is_at_capacity(self):
         比较耗时 半小时检查一次
         @return:
         """
-        # if self._is_at_capacity:
-        #     return self._is_at_capacity
-        #
-        # if not self._check_capacity_time or time.time() - self._check_capacity_time > 1800:
-        #     bit_count = self.bitarray.count()
-        #     if bit_count and bit_count / self.num_bits > 0.5:
-        #         self._is_at_capacity = True
-        #
-        #     self._check_capacity_time = time.time()
-        #
-        # return self._is_at_capacity
-
         if self._is_at_capacity:
             return self._is_at_capacity
 
-        bit_count = self.bitarray.count()
-        if bit_count and bit_count / self.num_bits > 0.5:
-            self._is_at_capacity = True
+        if (
+            not self._check_capacity_time
+            or time.time() - self._check_capacity_time > 1800
+        ):
+            bit_count = self.bitarray.count()
+            if bit_count and bit_count / self.num_bits > 0.5:
+                self._is_at_capacity = True
+
+            self._check_capacity_time = time.time()
 
         return self._is_at_capacity
 
@@ -173,8 +167,8 @@ def add(self, keys):
         @param keys: list or one key
         @return:
         """
-        if self.is_at_capacity:
-            raise IndexError("BloomFilter is at capacity")
+        # if self.is_at_capacity:
+        #     raise IndexError("BloomFilter is at capacity")
 
         is_list = isinstance(keys, list)
 

diff --git a/tests/test_dedup.py b/tests/test_dedup.py
@@ -54,3 +54,10 @@ def test_filter():
     datas = ["xxx", "bbb", "ccc"]
     dedup.filter_exist_data(datas)
     assert datas == ["ccc"]
+
+def test_ScalableBloomFilter():
+    dedup = Dedup(Dedup.BloomFilter, redis_url="redis://@localhost:6379/0", initial_capacity=10)
+    for i in range(1000):
+        print(dedup.add(i))
+
+test_ScalableBloomFilter()