修复代理池问题

feapder/VERSION

@@ -1 +1 @@
-1.4.1
+1.4.2

feapder/network/proxy_pool.py

@@ -14,8 +14,8 @@
 import requests
 
 from feapder import setting
-from feapder.utils import log
 from feapder.utils import tools
+from feapder.utils.log import log
 
 # 建立本地缓存代理文件夹
 proxy_path = os.path.join(os.path.dirname(__file__), "proxy_file")
@@ -174,7 +174,7 @@ def check_proxy(
     type=0,
     timeout=5,
     logger=None,
-    show_error_log=False,
+    show_error_log=True,
     **kwargs,
 ):
     """
@@ -187,7 +187,7 @@ def check_proxy(
     :return:
     """
     if not logger:
-        logger = log.get_logger(__file__)
+        logger = log
     ok = 0
     if type == 0 and ip and port:
         # socket检测成功 不代表代理一定可用 Connection closed by foreign host. 这种情况就不行
@@ -207,21 +207,9 @@ def check_proxy(
                 "http": "http://{}:{}".format(ip, port),
                 "https": "https://{}:{}".format(ip, port),
             }
-        target_url = random.choice(
-            [
-                "http://www.baidu.com",
-                # "http://httpbin.org/ip",
-            ]
-        )
         try:
             r = requests.get(
-                target_url,
-                headers={
-                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"
-                },
-                proxies=proxies,
-                timeout=timeout,
-                stream=True,
+                "http://www.baidu.com", proxies=proxies, timeout=timeout, stream=True
             )
             ok = 1
             r.close()
@@ -247,7 +235,6 @@ def __init__(
         max_proxy_use_num=10000,
         delay=30,
         use_interval=None,
-        logger=None,
         **kwargs,
     ):
         """
@@ -294,7 +281,7 @@ def __init__(
             self.proxy_id = self.proxy_ip_port
 
         # 日志处理器
-        self.logger = logger or log.get_logger(__file__)
+        self.logger = log
 
     def get_proxies(self):
         self.use_num += 1
@@ -418,7 +405,7 @@ def __init__(self, **kwargs):
             self.proxy_source_url = list(set(self.proxy_source_url))
             kwargs.update({"proxy_source_url": self.proxy_source_url})
         # 处理日志
-        self.logger = kwargs.get("logger") or log.get_logger(__file__)
+        self.logger = kwargs.get("logger") or log
         kwargs["logger"] = self.logger
         if not self.proxy_source_url:
             self.logger.warn("need set proxy_source_url or proxy_instance")

tests/test_proxies_pool.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2021/4/3 4:25 下午
+---------
+@summary:
+---------
+@author: Boris
+@email: [email protected]
+"""
+from feapder.network.proxy_pool import ProxyPool, check_proxy
+import requests
+
+url = "http://tunnel-api.apeyun.com/h?id=2020120800184471713&secret=3U1fEJPuabi3y2QJ&limit=10&format=txt&auth_mode=auto"
+
+proxy_pool = ProxyPool(size=-1, proxy_source_url=url)
+
+print(proxy_pool.get())
+#
+# headers = {
+#     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
+#     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+#     "Accept-Encoding": "gzip, deflate, br",
+#     "Accept-Language": "zh-CN,zh;q=0.9",
+#     "Connection": "keep-alive",
+# }
+#
+#
+# resp = requests.get(
+#     "http://www.baidu.com",
+#     headers=headers,
+#     proxies={
+#         "https": "https://182.106.136.67:13586",
+#         "http": "http://182.106.136.67:13586",
+#     },
+# )
+# print(resp.text)
+#
+# a = check_proxy("182.106.136.67", "13586", show_error_log=True, type=1)
+# print(a)