From 3199048520ebe3798c14cfa7362612b51d156b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 2 Jul 2020 20:10:08 +0200 Subject: [PATCH] Complete Azure Pipelines CI setup --- azure-pipelines.yml | 18 ++++++------------ tests/CrawlerRunner/ip_address.py | 15 ++++++++++++++- tests/mockserver.py | 5 ++--- tests/test_commands.py | 5 +++++ tests/test_crawler.py | 13 +++++++++++++ tests/test_feedexport.py | 8 +++++++- tests/test_proxy_connect.py | 3 +++ tests/test_spiderloader/__init__.py | 23 +++++++++++++++-------- tests/test_utils_asyncio.py | 7 ++++++- tox.ini | 18 +++++++++++++++--- 10 files changed, 86 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ffc4d549bff..710e4209092 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,29 +1,23 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - - +variables: + TOXENV: py pool: - vmImage: 'windows-2019' + vmImage: 'windows-latest' strategy: matrix: Python35: python.version: '3.5' - TOXENV: py35 + TOXENV: windows-pinned Python36: python.version: '3.6' - TOXENV: py36 Python37: python.version: '3.7' - TOXENV: py37 - + Python38: + python.version: '3.8' steps: - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' displayName: 'Use Python $(python.version)' - - script: | pip install -U tox twine wheel codecov tox diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 826374cd4d1..ea75bc3c9b3 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,7 +1,9 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names.client import createResolver +from twisted.names import cache, hosts as hostsModule, resolve +from twisted.names.client import Resolver +from twisted.python.runtime import platform from scrapy import Spider, Request from scrapy.crawler import CrawlerRunner @@ -10,6 +12,17 @@ from tests.mockserver import MockServer, MockDNSServer +# https://stackoverflow.com/a/32784190 +def createResolver(servers=None, resolvconf=None, hosts=None): + if hosts is None: + hosts = (b'/etc/hosts' if platform.getType() == 'posix' + else r'c:\windows\hosts') + theResolver = Resolver(resolvconf, servers) + hostResolver = hostsModule.Resolver(hosts) + L = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(L) + + class LocalhostSpider(Spider): name = "localhost_spider" diff --git a/tests/mockserver.py b/tests/mockserver.py index df30feab68a..1f40473bae4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -247,9 +247,8 @@ class MockDNSServer: def __enter__(self): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], stdout=PIPE, env=get_testenv()) - host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") - self.host = host - self.port = int(port) + self.host = '127.0.0.1' + self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1]) return self def __exit__(self, exc_type, exc_value, traceback): diff --git a/tests/test_commands.py b/tests/test_commands.py index 24a341759b7..ee0e4511ad0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ import json import optparse import os +import platform import subprocess import sys import tempfile @@ -10,6 +11,7 @@ from shutil import rmtree, copytree from tempfile import mkdtemp from threading import Timer +from unittest import skipIf from twisted.trial import unittest @@ -319,6 +321,9 @@ def start_requests(self): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 78704fb2c12..1a4cfe81319 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,7 @@ import subprocess import sys import warnings +from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -252,6 +253,9 @@ def test_crawler_runner_asyncio_enabled_true(self): }) @defer.inlineCallbacks + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -293,11 +297,17 @@ def test_simple(self): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -327,6 +337,9 @@ def test_reactor_poll(self): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e386442140a..f7b99756045 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -455,9 +455,15 @@ def _random_temp_filename(self): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ + def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + def printf_escape(string): + return string.replace('%', '%%') + FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - urljoin('file:', pathname2url(str(file_path))): feed + printf_escape(path_to_url(file_path)): feed for file_path, feed in FEEDS.items() } diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index eb4ecc91d9b..fc5658ae7ad 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,5 +1,6 @@ import json import os +import platform import re import sys from subprocess import Popen, PIPE @@ -59,6 +60,8 @@ def _wrong_credentials(proxy_url): @skipIf(sys.version_info < (3, 5, 4), "requires mitmproxy < 3.0.0, which these tests do not support") +@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), + "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d922c60595d..4929f1e3e09 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -20,13 +20,20 @@ module_dir = os.path.dirname(os.path.abspath(__file__)) +def _copytree(source, target): + try: + shutil.copytree(source, target) + except shutil.Error: + pass + + class SpiderLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') self.tmpdir = tempfile.mkdtemp() self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) self.spider_loader = SpiderLoader.from_settings(settings) @@ -124,7 +131,7 @@ def setUp(self): self.tmpdir = self.mktemp() os.mkdir(self.tmpdir) self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) @@ -134,8 +141,8 @@ def tearDown(self): def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -156,10 +163,10 @@ def test_dupename_warning(self): def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py')) - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 295323e4daa..a2114bd1841 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,4 +1,6 @@ -from unittest import TestCase +import platform +import sys +from unittest import skipIf, TestCase from pytest import mark @@ -12,6 +14,9 @@ def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tox.ini b/tox.ini index 27d21ade2f0..f729327ca87 100644 --- a/tox.ini +++ b/tox.ini @@ -63,14 +63,12 @@ basepython = pypy3 commands = py.test {posargs:--durations=10 docs scrapy tests} -[testenv:pinned] -basepython = python3 +[pinned] deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 - lxml==3.5.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 @@ -85,6 +83,20 @@ deps = botocore==1.3.23 Pillow==3.4.2 +[testenv:pinned] +basepython = python3 +deps = + {[pinned]deps} + lxml==3.5.0 + +[testenv:windows-pinned] +basepython = python3 +deps = + {[pinned]deps} + # First lxml version that includes a Windows wheel for Python 3.5, so we do + # not need to build lxml from sources in a CI Windows job: + lxml==3.8.0 + [testenv:extra-deps] deps = {[testenv]deps}