修复delete_keys的bug

Boris-code · Aug 9, 2022 · 47dcdac · 47dcdac
1 parent 44cbce5
commit 47dcdac
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 61 deletions.
diff --git a/feapder/core/scheduler.py b/feapder/core/scheduler.py
@@ -402,19 +402,19 @@ def check_task_status(self):
                 msg, level="error", message_prefix="《%s》爬虫导出数据失败" % (self._spider_name)
             )
 
-    def delete_tables(self, delete_tables_list):
-        if isinstance(delete_tables_list, bool):
-            delete_tables_list = [self._redis_key + "*"]
-        elif not isinstance(delete_tables_list, (list, tuple)):
-            delete_tables_list = [delete_tables_list]
-
-        for delete_tab in delete_tables_list:
-            if not delete_tab.startswith(self._redis_key):
-                delete_tab = self._redis_key + delete_tab
-            tables = self._redisdb.getkeys(delete_tab)
-            for table in tables:
-                log.debug("正在删除key %s" % table)
-                self._redisdb.clear(table)
+    def delete_tables(self, delete_keys):
+        if delete_keys == True:
+            delete_keys = [self._redis_key + "*"]
+        elif not isinstance(delete_keys, (list, tuple)):
+            delete_keys = [delete_keys]
+
+        for delete_key in delete_keys:
+            if not delete_key.startswith(self._redis_key):
+                delete_key = self._redis_key + delete_key
+            keys = self._redisdb.getkeys(delete_key)
+            for key in keys:
+                log.debug("正在删除key %s" % key)
+                self._redisdb.clear(key)
 
     def _stop_all_thread(self):
         self._request_buffer.stop()

diff --git a/feapder/core/spiders/batch_spider.py b/feapder/core/spiders/batch_spider.py
@@ -1223,22 +1223,6 @@ def update_task_batch(self, task_id, state=1, *args, **kwargs):
 
             return update_item
 
-    def delete_tables(self, delete_tables_list):
-        if isinstance(delete_tables_list, bool):
-            delete_tables_list = [self._redis_key + "*"]
-        elif not isinstance(delete_tables_list, (list, tuple)):
-            delete_tables_list = [delete_tables_list]
-
-        redis = RedisDB()
-        for delete_tab in delete_tables_list:
-            if delete_tab == "*":
-                delete_tab = self._redis_key + "*"
-
-            tables = redis.getkeys(delete_tab)
-            for table in tables:
-                log.debug("正在清理表 %s" % table)
-                redis.clear(table)
-
     def run(self):
         self.start_monitor_task()
 

diff --git a/feapder/core/spiders/spider.py b/feapder/core/spiders/spider.py
@@ -274,22 +274,6 @@ def __init__(self, request=None, request_dict=None, *args, **kwargs):
     def save_cached(self, request, response, table):
         pass
 
-    def delete_tables(self, delete_tables_list):
-        if isinstance(delete_tables_list, bool):
-            delete_tables_list = [self._redis_key + "*"]
-        elif not isinstance(delete_tables_list, (list, tuple)):
-            delete_tables_list = [delete_tables_list]
-
-        redis = RedisDB()
-        for delete_tab in delete_tables_list:
-            if delete_tab == "*":
-                delete_tab = self._redis_key + "*"
-
-            tables = redis.getkeys(delete_tab)
-            for table in tables:
-                log.debug("正在清理表 %s" % table)
-                redis.clear(table)
-
     def __start_requests(self):
         yield self._request
 

diff --git a/feapder/core/spiders/task_spider.py b/feapder/core/spiders/task_spider.py
@@ -735,22 +735,6 @@ def update_task_batch(self, task_id, state=1, *args, **kwargs):
 
             return update_item
 
-    def delete_tables(self, delete_tables_list):
-        if isinstance(delete_tables_list, bool):
-            delete_tables_list = [self._redis_key + "*"]
-        elif not isinstance(delete_tables_list, (list, tuple)):
-            delete_tables_list = [delete_tables_list]
-
-        redis = RedisDB()
-        for delete_tab in delete_tables_list:
-            if delete_tab == "*":
-                delete_tab = self._redis_key + "*"
-
-            tables = redis.getkeys(delete_tab)
-            for table in tables:
-                log.debug("正在清理表 %s" % table)
-                redis.clear(table)
-
     def run(self):
         self.start_monitor_task()