将bitarray库拆出来，选择性安装

README.md

@@ -33,17 +33,23 @@
 
 From PyPi:
 
+通用版
+
 ```shell
 pip3 install feapder
 ```    
 
-From Git:
+完整版：
 
 ```shell
-pip3 install git+https://github.com/Boris-code/feapder.git
-```
+pip3 install feapder[all]
+``` 
+
+常用版与完整版区别：
+
+1. 完整版支持基于内存去重
 
-若安装出错，请参考[安装问题](https://boris.org.cn/feapder/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98)
+完整版可能会安装出错，若安装出错，请参考[安装问题](https://boris.org.cn/feapder/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98)
 
 ## 小试一下
 

docs/README.md

@@ -33,17 +33,23 @@
 
 From PyPi:
 
+通用版
+
 ```shell
 pip3 install feapder
 ```    
 
-From Git:
+完整版：
 
 ```shell
-pip3 install git+https://github.com/Boris-code/feapder.git
-```
+pip3 install feapder[all]
+``` 
+
+常用版与完整版区别：
+
+1. 完整版支持基于内存去重
 
-若安装出错，请参考[安装问题](question/安装问题)
+完整版可能会安装出错，若安装出错，请参考[安装问题](question/安装问题)
 
 ## 小试一下
 

feapder/VERSION

@@ -1 +1 @@
-1.3.9
+1.4.0

feapder/dedup/bitarray.py

@@ -10,7 +10,6 @@
 
 from __future__ import absolute_import
 
-import bitarray
 
 from feapder.db.redisdb import RedisDB
 
@@ -45,6 +44,13 @@ def count(self, value=True):
 
 class MemoryBitArray(BitArray):
     def __init__(self, num_bits):
+        try:
+            import bitarray
+        except Exception as e:
+            raise Exception(
+                "需要安装feapder完整版\ncommand: pip install feapder[all]\n若安装出错，参考：https://boris.org.cn/feapder/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98"
+            )
+
         self.num_bits = num_bits
         self.bitarray = bitarray.bitarray(num_bits, endian="little")
 

setup.py

@@ -33,31 +33,36 @@
     ]
 )
 
+requires = [
+    "better-exceptions>=0.2.2",
+    "DBUtils>=2.0",
+    "parsel>=1.5.2",
+    "PyExecJS>=1.5.1",
+    "PyMySQL>=0.9.3",
+    "redis>=2.10.6",
+    "requests>=2.22.0",
+    "bs4>=0.0.1",
+    "ipython>=7.14.0",
+    "redis-py-cluster>=1.3.4",
+    "cryptography>=3.3.2",
+    "selenium>=3.141.0",
+]
+
+memory_dedup_requires = ["bitarray>=1.5.3"]
+all_requires = memory_dedup_requires
+
 setuptools.setup(
     name="feapder",
     version=version,
     author="Boris",
     license="MIT",
     author_email="[email protected]",
-    python_requires='>=3.6',
+    python_requires=">=3.6",
     description="feapder是一款支持分布式、批次采集、任务防丢、报警丰富的python爬虫框架",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    install_requires=[  # 工具包的依赖包
-        "better-exceptions>=0.2.2",
-        "DBUtils>=2.0",
-        "parsel>=1.5.2",
-        "PyExecJS>=1.5.1",
-        "PyMySQL>=0.9.3",
-        "redis>=2.10.6",
-        "requests>=2.22.0",
-        "bs4>=0.0.1",
-        "ipython>=7.14.0",
-        "bitarray>=1.5.3",
-        "redis-py-cluster>=1.3.4",
-        "cryptography>=3.3.2",
-        "selenium>=3.141.0"
-    ],
+    install_requires=requires,
+    extras_require={"all": all_requires},
     entry_points={"console_scripts": ["feapder = feapder.commands.cmdline:execute"]},
     url="https://github.com/Boris-code/feapder.git",
     packages=packages,