diff --git a/README.md b/README.md index f022c074..b21ac869 100644 --- a/README.md +++ b/README.md @@ -33,17 +33,23 @@ From PyPi: +通用版 + ```shell pip3 install feapder ``` -From Git: +完整版: ```shell -pip3 install git+https://github.com/Boris-code/feapder.git -``` +pip3 install feapder[all] +``` + +常用版与完整版区别: + +1. 完整版支持基于内存去重 -若安装出错,请参考[安装问题](https://boris.org.cn/feapder/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98) +完整版可能会安装出错,若安装出错,请参考[安装问题](https://boris.org.cn/feapder/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98) ## 小试一下 diff --git a/docs/README.md b/docs/README.md index d78623db..b4e2844d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -33,17 +33,23 @@ From PyPi: +通用版 + ```shell pip3 install feapder ``` -From Git: +完整版: ```shell -pip3 install git+https://github.com/Boris-code/feapder.git -``` +pip3 install feapder[all] +``` + +常用版与完整版区别: + +1. 完整版支持基于内存去重 -若安装出错,请参考[安装问题](question/安装问题) +完整版可能会安装出错,若安装出错,请参考[安装问题](question/安装问题) ## 小试一下 diff --git a/feapder/VERSION b/feapder/VERSION index d4c4950a..88c5fb89 100644 --- a/feapder/VERSION +++ b/feapder/VERSION @@ -1 +1 @@ -1.3.9 +1.4.0 diff --git a/feapder/dedup/bitarray.py b/feapder/dedup/bitarray.py index b24c444d..0539ddde 100644 --- a/feapder/dedup/bitarray.py +++ b/feapder/dedup/bitarray.py @@ -10,7 +10,6 @@ from __future__ import absolute_import -import bitarray from feapder.db.redisdb import RedisDB @@ -45,6 +44,13 @@ def count(self, value=True): class MemoryBitArray(BitArray): def __init__(self, num_bits): + try: + import bitarray + except Exception as e: + raise Exception( + "需要安装feapder完整版\ncommand: pip install feapder[all]\n若安装出错,参考:https://boris.org.cn/feapder/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98" + ) + self.num_bits = num_bits self.bitarray = bitarray.bitarray(num_bits, endian="little") diff --git a/setup.py b/setup.py index 1ca13a92..f58de0ab 100644 --- a/setup.py +++ b/setup.py @@ -33,31 +33,36 @@ ] ) +requires = [ + "better-exceptions>=0.2.2", + "DBUtils>=2.0", + "parsel>=1.5.2", + "PyExecJS>=1.5.1", + "PyMySQL>=0.9.3", + "redis>=2.10.6", + "requests>=2.22.0", + "bs4>=0.0.1", + "ipython>=7.14.0", + "redis-py-cluster>=1.3.4", + "cryptography>=3.3.2", + "selenium>=3.141.0", +] + +memory_dedup_requires = ["bitarray>=1.5.3"] +all_requires = memory_dedup_requires + setuptools.setup( name="feapder", version=version, author="Boris", license="MIT", author_email="feapder@qq.com", - python_requires='>=3.6', + python_requires=">=3.6", description="feapder是一款支持分布式、批次采集、任务防丢、报警丰富的python爬虫框架", long_description=long_description, long_description_content_type="text/markdown", - install_requires=[ # 工具包的依赖包 - "better-exceptions>=0.2.2", - "DBUtils>=2.0", - "parsel>=1.5.2", - "PyExecJS>=1.5.1", - "PyMySQL>=0.9.3", - "redis>=2.10.6", - "requests>=2.22.0", - "bs4>=0.0.1", - "ipython>=7.14.0", - "bitarray>=1.5.3", - "redis-py-cluster>=1.3.4", - "cryptography>=3.3.2", - "selenium>=3.141.0" - ], + install_requires=requires, + extras_require={"all": all_requires}, entry_points={"console_scripts": ["feapder = feapder.commands.cmdline:execute"]}, url="https://github.com/Boris-code/feapder.git", packages=packages,