tox.ini

# Tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.

[tox]
envlist = security,flake8,py
minversion = 1.7.0

[testenv]
deps =
    -ctests/constraints.txt
    -rtests/requirements-py3.txt
    # mitmproxy does not support PyPy
    # mitmproxy does not support Windows when running Python < 3.7
    mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
    mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
    # Extras
    botocore>=1.4.87
    Pillow>=4.0.0
passenv =
    S3_TEST_FILE_URI
    AWS_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY
    GCS_TEST_FILE_URI
    GCS_PROJECT_ID
#allow tox virtualenv to upgrade pip/wheel/setuptools
download = true 
commands =
    py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}

[testenv:typing]
basepython = python3
deps =
    mypy==0.780
commands =
    mypy {posargs: scrapy tests}

[testenv:security]
basepython = python3
deps =
    bandit
commands =
    bandit -r -c .bandit.yml {posargs:scrapy}

[testenv:flake8]
basepython = python3
deps =
    {[testenv]deps}
    pytest-flake8
commands =
    py.test --flake8 {posargs:docs scrapy tests}

[testenv:pylint]
basepython = python3
deps =
    {[testenv]deps}
    # Optional dependencies
    boto
    reppy
    robotexclusionrulesparser
    # Test dependencies
    pylint
commands =
    pylint conftest.py docs extras scrapy setup.py tests

[pinned]
deps =
    -ctests/constraints.txt
    cryptography==2.0
    cssselect==0.9.1
    itemadapter==0.1.0
    parsel==1.5.0
    Protego==0.1.15
    pyOpenSSL==16.2.0
    queuelib==1.4.2
    service_identity==16.0.0
    Twisted==17.9.0
    w3lib==1.17.0
    zope.interface==4.1.3
    -rtests/requirements-py3.txt

    # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
    # above, hence we do not install it in pinned environments at the moment

    # Extras
    botocore==1.4.87
    google-cloud-storage==1.29.0
    Pillow==4.0.0
install_command =
    # --use-feature=2020-resolver is required, otherwise the latest verion of
    # Twisted gets installed.
    pip install --use-feature=2020-resolver {opts} {packages}
setenv =
    _SCRAPY_PINNED=true

[testenv:pinned]
deps =
    {[pinned]deps}
    lxml==3.5.0
    PyDispatcher==2.0.5
install_command =
    {[pinned]install_command}
setenv =
    {[pinned]setenv}
    
[testenv:windows-pinned]
basepython = python3
deps =
    {[pinned]deps}
    # First lxml version that includes a Windows wheel for Python 3.6, so we do
    # not need to build lxml from sources in a CI Windows job:
    lxml==3.8.0
    PyDispatcher==2.0.5
install_command =
    {[pinned]install_command}
setenv =
    {[pinned]setenv}

[testenv:extra-deps]
deps =
    {[testenv]deps}
    reppy
    robotexclusionrulesparser
install_command =
    # Test --use-feature=2020-resolver for the latest version of all
    # dependencies.
    pip install --use-feature=2020-resolver {opts} {packages}

[testenv:asyncio]
commands =
    {[testenv]commands} --reactor=asyncio

[testenv:asyncio-pinned]
deps = {[testenv:pinned]deps}
install_command =
    {[pinned]install_command}
commands = {[testenv:asyncio]commands}
setenv =
    {[pinned]setenv}

[testenv:pypy3]
basepython = pypy3
commands =
    py.test {posargs:--durations=10 docs scrapy tests}

[testenv:pypy3-pinned]
basepython = {[testenv:pypy3]basepython}
deps =
    {[pinned]deps}
    lxml==4.0.0
    PyPyDispatcher==2.1.0
install_command =
    {[pinned]install_command}
commands = {[testenv:pypy3]commands}
setenv =
    {[pinned]setenv}

[docs]
changedir = docs
deps =
    -rdocs/requirements.txt
setenv =
    READTHEDOCS_PROJECT=scrapy
    READTHEDOCS_VERSION=master

[testenv:docs]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
    sphinx-build -W -b html . {envtmpdir}/html

[testenv:docs-coverage]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
    sphinx-build -b coverage . {envtmpdir}/coverage

[testenv:docs-links]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
    sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck