Merge branch 'develop'

feapder/core/spiders/batch_spider.py

@@ -26,6 +26,7 @@
 from feapder.network.item import UpdateItem
 from feapder.network.request import Request
 from feapder.utils.log import log
+from feapder.utils.perfect_dict import PerfectDict
 from feapder.utils.redis_lock import RedisLock
 
 CONSOLE_PIPELINE_PATH = "feapder.pipelines.console_pipeline.ConsolePipeline"
@@ -111,7 +112,7 @@ def __init__(
             auto_start_requests=False,
             send_run_time=send_run_time,
             batch_interval=batch_interval,
-            task_table=task_table
+            task_table=task_table,
         )
 
         self._redisdb = RedisDB()
@@ -325,6 +326,9 @@ def distribute_task(self, tasks):
             for task in tasks:
                 for parser in self._parsers:  # 寻找task对应的parser
                     if parser.name in task:
+                        task = PerfectDict(
+                            _dict=dict(zip(self._task_keys, task)), _values=list(task)
+                        )
                         requests = parser.start_requests(task)
                         if requests and not isinstance(requests, Iterable):
                             raise Exception(
@@ -372,6 +376,9 @@ def distribute_task(self, tasks):
         else:  # task没对应的parser 则将task下发到所有的parser
             for task in tasks:
                 for parser in self._parsers:
+                    task = PerfectDict(
+                        _dict=dict(zip(self._task_keys, task)), _values=list(task)
+                    )
                     requests = parser.start_requests(task)
                     if requests and not isinstance(requests, Iterable):
                         raise Exception(

feapder/network/request.py

@@ -9,7 +9,6 @@
 """
 
 import requests
-from feapder.utils.webdriver import WebDriverPool
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
@@ -21,6 +20,8 @@
 from feapder.network.proxy_pool import proxy_pool
 from feapder.network.response import Response
 from feapder.utils.log import log
+from feapder.utils.perfect_dict import PerfectDict
+from feapder.utils.webdriver import WebDriverPool
 
 # 屏蔽warning信息
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
@@ -215,7 +216,7 @@ def to_dict(self):
             ):
                 continue
 
-            if callable(value) or isinstance(value, Item):  # 序列化 如item
+            if callable(value) or isinstance(value, (Item, PerfectDict)):
                 value = tools.dumps_obj(value)
 
             request_dict[key] = value
@@ -254,10 +255,7 @@ def get_response(self, save_cached=False):
                 self.requests_kwargs.update(headers=headers)
         else:
             self.requests_kwargs.setdefault(
-                "headers",
-                {
-                    "User-Agent": setting.DEFAULT_USERAGENT
-                },
+                "headers", {"User-Agent": setting.DEFAULT_USERAGENT}
             )
 
         # 代理

feapder/network/response.py

@@ -243,8 +243,9 @@ def text(self):
             else:
                 self._cached_text = self._get_unicode_html(self.content)
 
-            self._cached_text = self._absolute_links(self._cached_text)
-            self._cached_text = self._del_special_character(self._cached_text)
+            if self._cached_text:
+                self._cached_text = self._absolute_links(self._cached_text)
+                self._cached_text = self._del_special_character(self._cached_text)
 
         return self._cached_text
 

feapder/utils/perfect_dict.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2021/4/8 11:32 上午
+---------
+@summary:
+---------
+@author: Boris
+@email: [email protected]
+"""
+
+
+class PerfectDict(dict):
+    """
+    >>> from feapder.utils.perfect_dict import PerfectDict
+    >>> data = PerfectDict(id=1, url="xxx")
+    >>> data
+    {'id': 1, 'url': 'xxx'}
+    >>> data.id
+    1
+    >>> data.get("id")
+    1
+    >>> data["id"]
+    1
+    >>> id, url = data
+    >>> id
+    1
+    >>> url
+    'xxx'
+    >>> data[0]
+    1
+    >>> data[1]
+    'xxx'
+    >>> data = PerfectDict({"id":1, "url":"xxx"})
+    >>> data
+    {'id': 1, 'url': 'xxx'}
+    """
+
+    def __init__(self, _dict: dict = None, _values: list = None, **kwargs):
+        self.__dict__ = _dict or kwargs or {}
+        super().__init__(self.__dict__, **kwargs)
+        self.__values__ = _values or list(self.__dict__.values())
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.__values__[key]
+        else:
+            return self.__dict__[key]
+
+    def __iter__(self, *args, **kwargs):
+        for value in self.__values__:
+            yield value

tests/batch-spider/spiders/test_spider.py

@@ -49,6 +49,4 @@ def failed_request(self, request, response):
         """
 
         yield request
-        yield self.update_task_batch(request.task_id, -1) # 更新任务状态为-1
-
-
+        yield self.update_task_batch(request.task_id, -1)  # 更新任务状态为-1

tests/test_task.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2021/4/8 1:06 下午
+---------
+@summary:
+---------
+@author: Boris
+@email: [email protected]
+"""
+
+from feapder.utils.perfect_dict import PerfectDict
+
+
+task_key = ["id", "url"]
+task = [1, "http://www.badu.com"]
+task = Task(_dict=dict(zip(task_key, task)), _values=task)
+
+task = Task(id=1, url="http://www.badu.com")
+task = Task({"id":"1", "url":"http://www.badu.com"})
+
+print(task)
+id, url = task
+print(id, url)
+print(task[0], task[1])
+print(task.id, task.url)
+print(task["id"], task["url"])
+print(task.get("id"), task.get("url"))