From 1f22ac0286396cb6c2d370af9a538b4b97aefa25 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 10 Jan 2023 16:37:34 +0100 Subject: [PATCH 1/9] nomore pylib --- .pre-commit-config.yaml | 4 +- doc/example/conftest.py | 21 +-- doc/example/svn-sync-repo.py | 127 ---------------- doc/example/sysinfo.py | 16 +-- execnet/gateway_base.py | 3 +- execnet/gateway_io.py | 27 +--- execnet/multi.py | 14 +- pyproject.toml | 4 + testing/conftest.py | 53 +++---- testing/test_basics.py | 271 ++++++++++++++++++----------------- testing/test_gateway.py | 11 +- testing/test_multi.py | 1 - testing/test_rsync.py | 11 +- testing/test_serializer.py | 123 ++++++---------- testing/test_termination.py | 18 +-- testing/test_threadpool.py | 6 +- testing/test_xspec.py | 1 - tox.ini | 5 +- 18 files changed, 269 insertions(+), 447 deletions(-) delete mode 100644 doc/example/svn-sync-repo.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 107c01a0..0926b358 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: doc/en/example/py2py3/test_py2.py + repos: - repo: https://github.com/codespell-project/codespell rev: v2.2.2 @@ -28,7 +28,7 @@ repos: rev: v3.9.0 hooks: - id: reorder-python-imports - args: ['--application-directories=execnet', --py37-plus] + args: ['--application-directories=execnet', --py37-plus, "--remove-import=import py"] - repo: https://github.com/PyCQA/doc8 rev: 'v1.1.1' hooks: diff --git a/doc/example/conftest.py b/doc/example/conftest.py index 5ea2370e..7f54ba58 100644 --- a/doc/example/conftest.py +++ b/doc/example/conftest.py @@ -1,14 +1,19 @@ +import os import sys +from pathlib import Path -import py +def _add_path(path: Path): + strpath = os.fspath(path) + if strpath not in sys.path: + sys.path.insert(0, strpath) + + +mydir = Path(__file__).parent # make execnet and example code importable -cand = py.path.local(__file__).dirpath().dirpath().dirpath() -if cand.join("execnet", "__init__.py").check(): - if str(cand) not in sys.path: - sys.path.insert(0, str(cand)) -cand = py.path.local(__file__).dirpath() -if str(cand) not in sys.path: - sys.path.insert(0, str(cand)) +cand = mydir.parent.parent +if cand.joinpath("execnet", "__init__.py").is_file(): + _add_path(cand) +_add_path(mydir) pytest_plugins = ["doctest"] diff --git a/doc/example/svn-sync-repo.py b/doc/example/svn-sync-repo.py deleted file mode 100644 index 69c2dcb2..00000000 --- a/doc/example/svn-sync-repo.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python -""" - -small utility for hot-syncing a svn repository through ssh. -uses execnet. - -""" -import os -import sys - -import execnet -import py - - -def usage(): - arg0 = sys.argv[0] - print(arg0, "[user@]remote-host:/repo/location localrepo [ssh-config-file]") - - -def main(args): - remote = args[0] - localrepo = py.path.local(args[1]) - if not localrepo.check(dir=1): - raise SystemExit(f"localrepo {localrepo} does not exist") - if len(args) == 3: - configfile = args[2] - else: - configfile = None - remote_host, path = remote.split(":", 1) - print("ssh-connecting to", remote_host) - gw = getgateway(remote_host, configfile) - - local_rev = get_svn_youngest(localrepo) - - # local protocol - # 1. client sends rev/repo -> server - # 2. server checks for newer revisions and sends dumps - # 3. client receives dumps, updates local repo - # 4. client goes back to step 1 - c = gw.remote_exec( - """ - import py - import os - import time - remote_rev, repopath = channel.receive() - while 1: - rev = py.process.cmdexec('svnlook youngest "%s"' % repopath) - rev = int(rev) - if rev > remote_rev: - revrange = (remote_rev+1, rev) - dumpchannel = channel.gateway.newchannel() - channel.send(revrange) - channel.send(dumpchannel) - - f = os.popen( - "svnadmin dump -q --incremental -r %s:%s %s" - % (revrange[0], revrange[1], repopath), 'r') - try: - maxcount = dumpchannel.receive() - count = maxcount - while 1: - s = f.read(8192) - if not s: - raise EOFError - dumpchannel.send(s) - count = count - 1 - if count <= 0: - ack = dumpchannel.receive() - count = maxcount - - except EOFError: - dumpchannel.close() - remote_rev = rev - else: - # using svn-hook instead would be nice here - time.sleep(30) - """ - ) - - c.send((local_rev, path)) - print("checking revisions from %d in %s" % (local_rev, remote)) - while 1: - revstart, revend = c.receive() - dumpchannel = c.receive() - print("receiving revisions", revstart, "-", revend, "replaying...") - svn_load(localrepo, dumpchannel) - print("current revision", revend) - - -def svn_load(repo, dumpchannel, maxcount=100): - # every maxcount we will send an ACK to the other - # side in order to synchronise and avoid our side - # growing buffers (execnet does not control - # RAM usage or receive queue sizes) - dumpchannel.send(maxcount) - f = os.popen(f"svnadmin load -q {repo}", "w") - count = maxcount - for x in dumpchannel: - sys.stdout.write(".") - sys.stdout.flush() - f.write(x) - count = count - 1 - if count <= 0: - dumpchannel.send(maxcount) - count = maxcount - print >> sys.stdout - f.close() - - -def get_svn_youngest(repo): - rev = py.process.cmdexec('svnlook youngest "%s"' % repo) - return int(rev) - - -def getgateway(host, configfile=None): - xspec = "ssh=%s" % host - if configfile is not None: - xspec += "//ssh_config=%s" % configfile - return execnet.makegateway(xspec) - - -if __name__ == "__main__": - if len(sys.argv) < 3: - usage() - raise SystemExit(1) - - main(sys.argv[1:]) diff --git a/doc/example/sysinfo.py b/doc/example/sysinfo.py index 99a523b5..d1eea34f 100644 --- a/doc/example/sysinfo.py +++ b/doc/example/sysinfo.py @@ -10,7 +10,6 @@ import sys import execnet -import py parser = optparse.OptionParser(usage=__doc__) @@ -34,14 +33,15 @@ def parsehosts(path): - path = py.path.local(path) + host_regex = re.compile(r"Host\s*(\S+)") l = [] - rex = re.compile(r"Host\s*(\S+)") - for line in path.readlines(): - m = rex.match(line) - if m is not None: - (sshname,) = m.groups() - l.append(sshname) + + with open(path) as fp: + for line in fp: + m = rex.match(line) + if m is not None: + (sshname,) = m.groups() + l.append(sshname) return l diff --git a/execnet/gateway_base.py b/execnet/gateway_base.py index fd3c7f63..9577a94b 100644 --- a/execnet/gateway_base.py +++ b/execnet/gateway_base.py @@ -20,6 +20,7 @@ import weakref from io import BytesIO from typing import Callable +from typing import IO def reraise(cls, val, tb): @@ -961,7 +962,7 @@ def log(*msg): def _terminate_execution(self): pass - def _send(self, msgcode, channelid=0, data=b""): + def _send(self, msgcode, channelid: int = 0, data: bytes = b""): message = Message(msgcode, channelid, data) try: message.to_io(self._io) diff --git a/execnet/gateway_io.py b/execnet/gateway_io.py index 1e49543f..84352655 100644 --- a/execnet/gateway_io.py +++ b/execnet/gateway_io.py @@ -32,29 +32,12 @@ def kill(self): def killpopen(popen): try: - if hasattr(popen, "kill"): - popen.kill() - else: - killpid(popen.pid) - except OSError: - sys.stderr.write("ERROR killing: %s\n" % (sys.exc_info()[1])) + popen.kill() + except OSError as e: + sys.stderr.write("ERROR killing: %s\n" % e) sys.stderr.flush() -def killpid(pid): - if hasattr(os, "kill"): - os.kill(pid, 15) - elif sys.platform == "win32" or getattr(os, "_name", None) == "nt": - import ctypes - - PROCESS_TERMINATE = 1 - handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, False, pid) - ctypes.windll.kernel32.TerminateProcess(handle, -1) - ctypes.windll.kernel32.CloseHandle(handle) - else: - raise OSError(f"no method to kill {pid}") - - popen_bootstrapline = "import sys;exec(eval(sys.stdin.readline()))" @@ -72,10 +55,8 @@ def shell_split_path(path): def popen_args(spec): args = shell_split_path(spec.python) if spec.python else [sys.executable] args.append("-u") - if spec is not None and spec.dont_write_bytecode: + if spec.dont_write_bytecode: args.append("-B") - # Slight gymnastics in ordering these arguments because CPython (as of - # 2.7.1) ignores -B if you provide `python -c "something" -B` args.extend(["-c", popen_bootstrapline]) return args diff --git a/execnet/multi.py b/execnet/multi.py index 838fd546..285851b5 100644 --- a/execnet/multi.py +++ b/execnet/multi.py @@ -3,6 +3,8 @@ (c) 2008-2014, Holger Krekel and others """ +from __future__ import annotations + import atexit import sys from functools import partial @@ -280,16 +282,16 @@ def putreceived(obj, channel=ch): ch.setcallback(putreceived, endmarker=endmarker) return self._queue - def waitclose(self): - first = None + def waitclose(self) -> None: + first: Exception | None = None for ch in self._channels: try: ch.waitclose() - except ch.RemoteError: + except ch.RemoteError as e: if first is None: - first = sys.exc_info() - if first: - reraise(*first) + first = e + if first is not None: + raise first def safe_terminate(execmodel, timeout, list_of_paired_functions): diff --git a/pyproject.toml b/pyproject.toml index 78819678..218fac81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,3 +55,7 @@ version-file = "execnet/_version.py" include = [ "/execnet", ] + + +[tool.mypy] +python_version = "3.7" diff --git a/testing/conftest.py b/testing/conftest.py index d5e84683..6b472576 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,8 +1,10 @@ import subprocess import sys +from functools import lru_cache +from typing import Callable +from typing import Iterator -import execnet -import py +import execnet.gateway import pytest from execnet.gateway_base import get_execmodel from execnet.gateway_base import WorkerPool @@ -22,10 +24,15 @@ def pytest_runtest_setup(item): @pytest.fixture -def makegateway(request): +def group_function() -> Iterator[execnet.Group]: group = execnet.Group() - request.addfinalizer(lambda: group.terminate(0.5)) - return group.makegateway + yield group + group.terminate(0.5) + + +@pytest.fixture +def makegateway(group_function) -> Callable[[str], execnet.gateway.Gateway]: + return group_function.makegateway pytest_plugins = ["pytester", "doctest"] @@ -76,7 +83,7 @@ def getspecssh(config): xspecs = getgspecs(config) for spec in xspecs: if spec.ssh: - if not py.path.local.sysfind("ssh"): + if not shutil.which("ssh"): pytest.skip("command not found: ssh") return spec pytest.skip("need '--gx ssh=...'") @@ -100,36 +107,18 @@ def pytest_generate_tests(metafunc): else: gwtypes = ["popen", "socket", "ssh", "proxy"] metafunc.parametrize("gw", gwtypes, indirect=True) - elif "anypython" in metafunc.fixturenames: - metafunc.parametrize( - "anypython", - indirect=True, - argvalues=("sys.executable", "pypy3"), - ) -def getexecutable(name, cache={}): - try: - return cache[name] - except KeyError: - if name == "sys.executable": - return py.path.local(sys.executable) - executable = py.path.local.sysfind(name) - if executable: - if name == "jython": - popen = subprocess.Popen( - [str(executable), "--version"], - universal_newlines=True, - stderr=subprocess.PIPE, - ) - out, err = popen.communicate() - if not err or "2.5" not in err: - executable = None - cache[name] = executable - return executable +@lru_cache() +def getexecutable(name): + if name == "sys.executable": + return sys.executable + import shutil + return shutil.which(name) -@pytest.fixture + +@pytest.fixture(params=("sys.executable", "pypy3")) def anypython(request): name = request.param executable = getexecutable(name) diff --git a/testing/test_basics.py b/testing/test_basics.py index 42eee1b7..afc292bb 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -1,11 +1,16 @@ +from __future__ import annotations + import inspect import os import subprocess import sys +import textwrap +from dataclasses import dataclass from io import BytesIO +from pathlib import Path +from typing import Any import execnet -import py import pytest from execnet import gateway from execnet import gateway_base @@ -21,28 +26,27 @@ ) +@pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: - pytestmark = [pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]])] - def test_serializer_api(self, val): dumped = execnet.dumps(val) val2 = execnet.loads(dumped) assert val == val2 - def test_mmap(self, tmpdir, val): + def test_mmap(self, tmp_path, val): mmap = pytest.importorskip("mmap").mmap - p = tmpdir.join("data") - with p.open("wb") as f: - f.write(execnet.dumps(val)) - f = p.open("r+b") - m = mmap(f.fileno(), 0) - val2 = execnet.load(m) + p = tmp_path / "data.bin" + + p.write_bytes(execnet.dumps(val)) + with p.open("r+b") as f: + m = mmap(f.fileno(), 0) + val2 = execnet.load(m) assert val == val2 def test_bytesio(self, val): - f = py.io.BytesIO() + f = BytesIO() execnet.dump(f, val) - read = py.io.BytesIO(f.getvalue()) + read = BytesIO(f.getvalue()) val2 = execnet.load(read) assert val == val2 @@ -75,14 +79,13 @@ def test_subprocess_interaction(anypython): def send(line): popen.stdin.write(line) - if sys.version_info > (3, 0) or sys.platform.startswith("java"): - popen.stdin.flush() + popen.stdin.flush() def receive(): return popen.stdout.readline() try: - source = py.code.Source(read_write_loop, "read_write_loop()") + source = f"{inspect.getsource(read_write_loop)}\n\nread_write_loop()" repr_source = repr(str(source)) + "\n" sendline = repr_source send(sendline) @@ -115,69 +118,83 @@ def read_write_loop(): break -def test_io_message(anypython, tmpdir, execmodel): - check = tmpdir.join("check.py") - check.write( - py.code.Source( - gateway_base, - """ - try: - from io import BytesIO - except ImportError: - from StringIO import StringIO as BytesIO - import tempfile - temp_out = BytesIO() - temp_in = BytesIO() - io = Popen2IO(temp_out, temp_in, get_execmodel({backend!r})) - for i, handler in enumerate(Message._types): - print ("checking %s %s" %(i, handler)) - for data in "hello", "hello".encode('ascii'): - msg1 = Message(i, i, dumps(data)) - msg1.to_io(io) - x = io.outfile.getvalue() - io.outfile.truncate(0) - io.outfile.seek(0) - io.infile.seek(0) - io.infile.write(x) - io.infile.seek(0) - msg2 = Message.from_io(io) - assert msg1.channelid == msg2.channelid, (msg1, msg2) - assert msg1.data == msg2.data, (msg1.data, msg2.data) - assert msg1.msgcode == msg2.msgcode - print ("all passed") - """.format( - backend=execmodel.backend - ), +IO_MESSAGE_EXTRA_SOURCE = """ +import sys +backend = sys.argv[1] +try: + from io import BytesIO +except ImportError: + from StringIO import StringIO as BytesIO +import tempfile +temp_out = BytesIO() +temp_in = BytesIO() +io = Popen2IO(temp_out, temp_in, get_execmodel(backend)) +for i, handler in enumerate(Message._types): + print ("checking", i, handler) + for data in "hello", "hello".encode('ascii'): + msg1 = Message(i, i, dumps(data)) + msg1.to_io(io) + x = io.outfile.getvalue() + io.outfile.truncate(0) + io.outfile.seek(0) + io.infile.seek(0) + io.infile.write(x) + io.infile.seek(0) + msg2 = Message.from_io(io) + assert msg1.channelid == msg2.channelid, (msg1, msg2) + assert msg1.data == msg2.data, (msg1.data, msg2.data) + assert msg1.msgcode == msg2.msgcode +print ("all passed") +""" + + +@dataclass +class Checker: + python: str + path: Path + idx: int = 0 + + def run_check( + self, script: str, *extra_args: str, **process_args: Any + ) -> subprocess.CompletedProcess[str]: + self.idx += 1 + check_path = self.path / f"check{self.idx}.py" + check_path.write_text(script) + return subprocess.run( + [self.python, os.fspath(check_path), *extra_args], + capture_output=True, + text=True, + check=True, + **process_args, ) + + +@pytest.fixture +def checker(anypython: str, tmp_path: Path) -> Checker: + return Checker(python=anypython, path=tmp_path) + + +def test_io_message(checker, execmodel): + out = checker.run_check( + inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE, execmodel.backend ) - # out = py.process.cmdexec("%s %s" %(executable,check)) - out = anypython.sysexec(check) - print(out) - assert "all passed" in out - - -def test_popen_io(anypython, tmpdir, execmodel): - check = tmpdir.join("check.py") - check.write( - py.code.Source( - gateway_base, - f""" - io = init_popen_io(get_execmodel({execmodel.backend!r})) - io.write("hello".encode('ascii')) - s = io.read(1) - assert s == "x".encode('ascii') - """, - ) + print(out.stdout) + assert "all passed" in out.stdout + + +def test_popen_io(checker, execmodel): + out = checker.run_check( + inspect.getsource(gateway_base) + + f""" +io = init_popen_io(get_execmodel({execmodel.backend!r})) +io.write(b"hello") +s = io.read(1) +assert s == b"x" +""", + input="x", ) - from subprocess import Popen, PIPE - - args = [str(anypython), str(check)] - proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) - proc.stdin.write(b"x") - stdout, stderr = proc.communicate() - print(stderr) - proc.wait() - assert b"hello" in stdout + print(out.stderr) + assert "hello" in out.stdout def test_popen_io_readloop(monkeypatch, execmodel): @@ -195,60 +212,50 @@ def newread(numbytes): assert result == b"tes" -def test_rinfo_source(anypython, tmpdir): - check = tmpdir.join("check.py") - check.write( - py.code.Source( - """ - class Channel: - def send(self, data): - assert eval(repr(data), {}) == data - channel = Channel() - """, - gateway.rinfo_source, - """ - print ('all passed') - """, - ) +def test_rinfo_source(checker): + out = checker.run_check( + f""" +class Channel: + def send(self, data): + assert eval(repr(data), {{}}) == data +channel = Channel() +{inspect.getsource(gateway.rinfo_source)} +print ('all passed') +""" ) - out = anypython.sysexec(check) - print(out) - assert "all passed" in out - - -def test_geterrortext(anypython, tmpdir): - check = tmpdir.join("check.py") - check.write( - py.code.Source( - gateway_base, - """ - class Arg: - pass - errortext = geterrortext((Arg, "1", 4)) - assert "Arg" in errortext - import sys - try: - raise ValueError("17") - except ValueError: - excinfo = sys.exc_info() - s = geterrortext(excinfo) - assert "17" in s - print ("all passed") - """, - ) + + print(out.stdout) + assert "all passed" in out.stdout + + +def test_geterrortext(checker): + out = checker.run_check( + inspect.getsource(gateway_base) + + """ +class Arg: + pass +errortext = geterrortext((Arg, "1", 4)) +assert "Arg" in errortext +import sys +try: + raise ValueError("17") +except ValueError: + excinfo = sys.exc_info() + s = geterrortext(excinfo) + assert "17" in s + print ("all passed") + """ ) - out = anypython.sysexec(check) - print(out) - assert "all passed" in out + print(out.stdout) + assert "all passed" in out.stdout @pytest.mark.skipif("not hasattr(os, 'dup')") -def test_stdouterrin_setnull(execmodel): - cap = py.io.StdCaptureFD() +def test_stdouterrin_setnull(execmodel, capfd): gateway_base.init_popen_io(execmodel) os.write(1, b"hello") os.read(0, 1) - out, err = cap.reset() + out, err = capfd.readouterr() assert not out assert not err @@ -271,7 +278,7 @@ def close(self, errortext=None): def test_exectask(execmodel): - io = py.io.BytesIO() + io = BytesIO() io.execmodel = execmodel gw = gateway_base.WorkerGateway(io, id="something") ch = PseudoChannel() @@ -282,10 +289,10 @@ def test_exectask(execmodel): class TestMessage: def test_wire_protocol(self): for i, handler in enumerate(Message._types): - one = py.io.BytesIO() + one = BytesIO() data = b"23" Message(i, 42, data).to_io(one) - two = py.io.BytesIO(one.getvalue()) + two = BytesIO(one.getvalue()) msg = Message.from_io(two) assert msg.msgcode == i assert isinstance(msg, Message) @@ -297,15 +304,15 @@ def test_wire_protocol(self): class TestPureChannel: @pytest.fixture def fac(self, execmodel): - class Gateway: + class FakeGateway: def _trace(self, *args): pass - def _send_(self, *k): + def _send(self, *k): pass - Gateway.execmodel = execmodel - return ChannelFactory(Gateway()) + FakeGateway.execmodel = execmodel + return ChannelFactory(FakeGateway()) def test_factory_create(self, fac): chan1 = fac.new() @@ -342,7 +349,7 @@ def prototype(wrong): def test_function_without_known_source_fails(self): # this one won't be able to find the source mess = {} - py.builtin.exec_("def fail(channel): pass", mess, mess) + exec("def fail(channel): pass", mess, mess) print(inspect.getsourcefile(mess["fail"])) pytest.raises(ValueError, gateway._source_of_function, mess["fail"]) @@ -365,9 +372,9 @@ def working(channel): class TestGlobalFinder: def check(self, func): - src = py.code.Source(func) - code = py.code.Code(func) - return gateway._find_non_builtin_globals(str(src), code.raw) + src = textwrap.dedent(inspect.getsource(func)) + code = func.__code__ + return gateway._find_non_builtin_globals(src, code) def test_local(self): def f(a, b, c): diff --git a/testing/test_gateway.py b/testing/test_gateway.py index d4b5847f..da498988 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -8,7 +8,6 @@ from textwrap import dedent import execnet -import py import pytest from execnet import gateway_base from execnet import gateway_io @@ -323,7 +322,7 @@ def test_waitclose_on_remote_killed(self, makegateway): """ ) remotepid = channel.receive() - py.process.kill(remotepid) + os.kill(remotepid, 15) with pytest.raises(EOFError): channel.waitclose(TESTTIMEOUT) with pytest.raises(IOError): @@ -454,10 +453,12 @@ def test_popen_filetracing(self, testdir, monkeypatch, makegateway): fn = gw.remote_exec( "import execnet;channel.send(execnet.gateway_base.fn)" ).receive() - workerfile = py.path.local(fn) - assert workerfile.check() + from pathlib import Path + + workerfile = Path(fn) + assert workerfile.is_file() worker_line = "creating workergateway" - for line in workerfile.readlines(): + for line in workerfile.read_text().splitlines(): if worker_line in line: break else: diff --git a/testing/test_multi.py b/testing/test_multi.py index 6504f3fb..79861b11 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -5,7 +5,6 @@ from time import sleep import execnet -import py import pytest from execnet import XSpec from execnet.gateway_base import Channel diff --git a/testing/test_rsync.py b/testing/test_rsync.py index ef7ba206..a90f6654 100644 --- a/testing/test_rsync.py +++ b/testing/test_rsync.py @@ -1,5 +1,4 @@ import execnet -import py import pytest from execnet import RSync from test_serializer import _find_version @@ -26,12 +25,6 @@ def gw2(request, group): return gw -needssymlink = pytest.mark.skipif( - not hasattr(py.path.local, "mksymlinkto"), - reason="py.path.local has no mksymlinkto() on this platform", -) - - @pytest.fixture def dirs(request, tmpdir): t = tmpdir @@ -156,7 +149,7 @@ def test_permissions(self, dirs, gw1, gw2): mode = destdir.stat().mode assert mode & 511 == 504 - @py.test.mark.skipif("sys.platform == 'win32' or getattr(os, '_name', '') == 'nt'") + @pytest.mark.skipif("sys.platform == 'win32' or getattr(os, '_name', '') == 'nt'") def test_read_only_directories(self, dirs, gw1): source = dirs.source dest = dirs.dest1 @@ -173,7 +166,6 @@ def test_read_only_directories(self, dirs, gw1): assert dest.join("sub").stat().mode & 0o700 assert dest.join("sub").join("subsub").stat().mode & 0o700 - @needssymlink def test_symlink_rsync(self, dirs, gw1): source = dirs.source dest = dirs.dest1 @@ -189,7 +181,6 @@ def test_symlink_rsync(self, dirs, gw1): assert dest.join("rellink").readlink() == "subdir/existent" assert dest.join("abslink").readlink() == expected - @needssymlink def test_symlink2_rsync(self, dirs, gw1): source = dirs.source dest = dirs.dest1 diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 594ac7c1..bb032461 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -1,10 +1,10 @@ import os +import shutil import subprocess import sys import tempfile import execnet -import py import pytest MINOR_VERSIONS = {"3": "543210", "2": "76"} @@ -12,107 +12,78 @@ def _find_version(suffix=""): name = "python" + suffix - executable = py.path.local.sysfind(name) - if executable is None: - if sys.platform == "win32" and suffix == "3": - for name in ("python31", "python30"): - executable = py.path.local(rf"c:\\{name}\python.exe") - if executable.check(): - return executable - for tail in MINOR_VERSIONS.get(suffix, ""): - path = py.path.local.sysfind(f"{name}.{tail}") - if path: - return path - - else: - pytest.skip(f"can't find a {name!r} executable") + path = shutil.which(name) + if path is not None: + return path + + for tail in MINOR_VERSIONS.get(suffix, ""): + path = shutil.which(f"{name}.{tail}") + if path: + return path + else: + pytest.skip(f"can't find a {name!r} executable") return executable -TEMPDIR = _py2_wrapper = _py3_wrapper = None +from pathlib import Path - -def setup_module(mod): - mod.TEMPDIR = py.path.local(tempfile.mkdtemp()) - mod._py3_wrapper = PythonWrapper(py.path.local(sys.executable)) - - -def teardown_module(mod): - TEMPDIR.remove(True) - - -# we use the execnet folder in order to avoid triggering a missing apipkg -pyimportdir = str(py.path.local(execnet.__file__).dirpath()) +# we use the execnet folder in order to avoid tiggering a missing apipkg +pyimportdir = os.fspath(Path(execnet.__file__).parent) class PythonWrapper: - def __init__(self, executable): + def __init__(self, executable, tmp_path): self.executable = executable + self.tmp_path = tmp_path - def dump(self, obj_rep): - script_file = TEMPDIR.join("dump.py") - script_file.write( - """ + def dump(self, obj_rep: str) -> bytes: + script_file = self.tmp_path.joinpath("dump.py") + script_file.write_text( + f""" import sys -sys.path.insert(0, %r) +sys.path.insert(0, {pyimportdir!r}) import gateway_base as serializer -if sys.version_info > (3, 0): # Need binary output - sys.stdout = sys.stdout.detach() -sys.stdout.write(serializer.dumps_internal(%s)) +sys.stdout = sys.stdout.detach() +sys.stdout.write(serializer.dumps_internal({obj_rep})) """ - % (pyimportdir, obj_rep) ) - popen = subprocess.Popen( - [str(self.executable), str(script_file)], - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + res = subprocess.run( + [self.executable, script_file], capture_output=True, check=True ) - stdout, stderr = popen.communicate() - ret = popen.returncode - if ret: - raise py.process.cmdexec.Error( - ret, ret, str(self.executable), stdout, stderr - ) - return stdout - - def load(self, data, option_args="__class__"): - script_file = TEMPDIR.join("load.py") - script_file.write( - r""" + return res.stdout + + def load(self, data: bytes): + script_file = self.tmp_path.joinpath("load.py") + script_file.write_text( + rf""" import sys -sys.path.insert(0, %r) +sys.path.insert(0, {pyimportdir!r}) import gateway_base as serializer -if sys.version_info > (3, 0): - sys.stdin = sys.stdin.detach() -loader = serializer.Unserializer(sys.stdin) -loader.%s +from io import BytesIO +data = {data!r} +io = BytesIO(data) +loader = serializer.Unserializer(io) obj = loader.load() sys.stdout.write(type(obj).__name__ + "\n") -sys.stdout.write(repr(obj))""" - % (pyimportdir, option_args) +sys.stdout.write(repr(obj)) +""" ) - popen = subprocess.Popen( - [str(self.executable), str(script_file)], - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + res = subprocess.run( + [self.executable, script_file], + capture_output=True, ) - stdout, stderr = popen.communicate(data) - ret = popen.returncode - if ret: - raise py.process.cmdexec.Error( - ret, ret, str(self.executable), stdout, stderr - ) - return [s.decode("ascii") for s in stdout.splitlines()] + if res.returncode: + raise ValueError(res.stderr) + + return res.stdout.decode("ascii").splitlines() def __repr__(self): return f"" @pytest.fixture -def py3(request): - return _py3_wrapper +def py3(request, tmp_path): + return PythonWrapper(sys.executable, tmp_path) @pytest.fixture diff --git a/testing/test_termination.py b/testing/test_termination.py index f0e484d1..ce496025 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -1,12 +1,14 @@ +import os import subprocess import sys +from pathlib import Path import execnet -import py import pytest from test_gateway import TESTTIMEOUT -execnetdir = py.path.local(execnet.__file__).dirpath().dirpath() + +execnetdir = Path(execnet.__file__).parent.parent skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -45,7 +47,7 @@ def test_endmarker_delivery_on_remote_killterm(makegateway, execmodel): """ ) pid = channel.receive() - py.process.kill(pid) + os.kill(pid, 15) channel.setcallback(q.put, endmarker=999) val = q.get(TESTTIMEOUT) assert val == 999 @@ -55,7 +57,9 @@ def test_endmarker_delivery_on_remote_killterm(makegateway, execmodel): @skip_win_pypy def test_termination_on_remote_channel_receive(monkeypatch, makegateway): - if not py.path.local.sysfind("ps"): + import shutil + + if not shutil.which("ps"): pytest.skip("need 'ps' command to externally check process status") monkeypatch.setenv("EXECNET_DEBUG", "2") gw = makegateway("popen") @@ -63,10 +67,8 @@ def test_termination_on_remote_channel_receive(monkeypatch, makegateway): gw.remote_exec("channel.receive()") gw._group.terminate() command = ["ps", "-p", str(pid)] - popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out, err = popen.communicate() - out = py.builtin._totext(out, "utf8") - assert str(pid) not in out, out + output = subprocess.run(command, capture_output=True, text=True) + assert str(pid) not in output.stdout, output def test_close_initiating_remote_no_error(testdir, anypython): diff --git a/testing/test_threadpool.py b/testing/test_threadpool.py index 2e537b57..d9b9ce0b 100644 --- a/testing/test_threadpool.py +++ b/testing/test_threadpool.py @@ -1,7 +1,6 @@ import os import sys -import py import pytest from execnet.gateway_base import WorkerPool @@ -143,8 +142,7 @@ def f(): @pytest.mark.skipif("not hasattr(os, 'dup')") -def test_pool_clean_shutdown(pool): - capture = py.io.StdCaptureFD() +def test_pool_clean_shutdown(pool, capfd): q = pool.execmodel.queue.Queue() def f(): @@ -162,7 +160,7 @@ def wait_then_put(): pool.execmodel.start(wait_then_put) assert pool.waitall() - out, err = capture.reset() + out, err = capfd.readouterr() sys.stdout.write(out + "\n") sys.stderr.write(err + "\n") assert err == "" diff --git a/testing/test_xspec.py b/testing/test_xspec.py index a75e5a1c..a841dfb6 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -6,7 +6,6 @@ import sys import execnet -import py import pytest from execnet.gateway_io import popen_args from execnet.gateway_io import ssh_args diff --git a/tox.ini b/tox.ini index be9c291b..b7a2db8a 100644 --- a/tox.ini +++ b/tox.ini @@ -2,12 +2,11 @@ envlist=py{37,38,39,310,311,pypy37},docs,linting isolated_build = true [testenv] +usedevelop=true deps= - setuptools_scm - py pytest pytest-timeout -passenv = GITHUB_ACTIONS +passenv = GITHUB_ACTIONS HOME USER, XDG_* commands= python -m pytest {posargs:testing} From 65280a74bea1c31e5d5156e5a107289e829ddc77 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 10 Jan 2023 22:05:01 +0100 Subject: [PATCH 2/9] all in on hatch + pre-commit update --- .github/workflows/main.yml | 23 +++++-- .gitignore | 3 +- .pre-commit-config.yaml | 4 ++ doc/example/test_funcmultiplier.py | 2 +- pyproject.toml | 68 ++++++++++++++----- {execnet => src/execnet}/__init__.py | 1 + {execnet => src/execnet}/gateway.py | 5 +- {execnet => src/execnet}/gateway_base.py | 0 {execnet => src/execnet}/gateway_bootstrap.py | 0 {execnet => src/execnet}/gateway_io.py | 0 {execnet => src/execnet}/gateway_socket.py | 0 {execnet => src/execnet}/multi.py | 0 {execnet => src/execnet}/rsync.py | 0 {execnet => src/execnet}/rsync_remote.py | 0 {execnet => src/execnet}/script/__init__.py | 0 .../execnet}/script/loop_socketserver.py | 0 {execnet => src/execnet}/script/quitserver.py | 0 {execnet => src/execnet}/script/shell.py | 0 .../execnet}/script/socketserver.py | 0 .../execnet}/script/socketserverservice.py | 10 +-- {execnet => src/execnet}/script/xx.py | 4 +- {execnet => src/execnet}/xspec.py | 20 ++++-- testing/test_gateway.py | 4 +- testing/test_serializer.py | 6 +- tox.ini | 31 --------- 25 files changed, 105 insertions(+), 76 deletions(-) rename {execnet => src/execnet}/__init__.py (96%) rename {execnet => src/execnet}/gateway.py (98%) rename {execnet => src/execnet}/gateway_base.py (100%) rename {execnet => src/execnet}/gateway_bootstrap.py (100%) rename {execnet => src/execnet}/gateway_io.py (100%) rename {execnet => src/execnet}/gateway_socket.py (100%) rename {execnet => src/execnet}/multi.py (100%) rename {execnet => src/execnet}/rsync.py (100%) rename {execnet => src/execnet}/rsync_remote.py (100%) rename {execnet => src/execnet}/script/__init__.py (100%) rename {execnet => src/execnet}/script/loop_socketserver.py (100%) rename {execnet => src/execnet}/script/quitserver.py (100%) rename {execnet => src/execnet}/script/shell.py (100%) rename {execnet => src/execnet}/script/socketserver.py (100%) rename {execnet => src/execnet}/script/socketserverservice.py (92%) rename {execnet => src/execnet}/script/xx.py (61%) rename {execnet => src/execnet}/xspec.py (78%) delete mode 100644 tox.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6503a864..14547db6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,15 @@ name: build -on: [push, pull_request] +on: + push: + branches: + - main + - master + - '*deploy*' + tags: + - v* + pull_request: + jobs: tests: @@ -10,8 +19,8 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest] - python: ["3.7","3.8","3.10","3.11", "pypy-3.7"] + os: [windows-latest, ubuntu-latest, osx-latest] + python: ["3.7","3.8","3.10","3.11", "pypy-3.7", "pypy-3.8"] steps: - uses: actions/checkout@v3 @@ -21,10 +30,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - name: Install tox - run: pip install tox + - name: Install hatch + run: pip install hatch + - id: pyname_fix + run: python -c "import sys;print('pyversion=' + sys.argv[1].replace('pypy-', 'pypy'))" ${{ matrix.python }} >> $GITHUB_OUTPUT - name: Test - run: tox -e py + run: hatch run +py=${{ steps.pyname_fix.outputs.pyversion }} test:test --color=yes deploy: diff --git a/.gitignore b/.gitignore index aafe44fc..d734b32f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ doc/_build build/ -execnet.egg-info/ -execnet/_version.py +src/execnet/_version.py dist/ .pytest_cache/ .eggs/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0926b358..00eb6421 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,3 +39,7 @@ repos: rev: 'v0.991' hooks: - id: mypy +- repo: https://github.com/tox-dev/pyproject-fmt + rev: "0.4.1" + hooks: + - id: pyproject-fmt diff --git a/doc/example/test_funcmultiplier.py b/doc/example/test_funcmultiplier.py index 6b26fb7c..bdd9a451 100644 --- a/doc/example/test_funcmultiplier.py +++ b/doc/example/test_funcmultiplier.py @@ -1,2 +1,2 @@ def test_function(): - import funcmultiplier + import funcmultiplier # type: ignore[import] diff --git a/pyproject.toml b/pyproject.toml index 218fac81..9403e1a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,22 @@ [build-system] +build-backend = "hatchling.build" requires = [ - "hatchling", - "hatch-vcs", + "hatch-vcs", + "hatchling", ] -build-backend = "hatchling.build" [project] name = "execnet" -dynamic = ["version"] description = "execnet: rapid multi-Python deployment" -long_description_file = "README.rst" +readme = "README.rst" license = "MIT" -requires-python = ">=3.7" authors = [ { name = "holger krekel and others" }, ] +requires-python = ">=3.7" +dynamic = [ + "version", +] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -33,29 +35,63 @@ classifiers = [ "Topic :: System :: Distributed Computing", "Topic :: System :: Networking", ] - [project.optional-dependencies] +dev = [ + "hatch", +] +docs = [ + "pyyaml", + "sphinx", +] testing = [ - "pre-commit", - "pytest", - "tox", - "hatch", + "pytest", + "pytest-timeout", ] [project.urls] Homepage = "https://execnet.readthedocs.io/en/latest/" + +[tool.pytest.ini_options] +timeout = 20 +addopts = "-ra" +testpaths = ["testing"] + + [tool.hatch.version] source = "vcs" +[tool.hatch.envs.test] +features = ["testing"] + +[[tool.hatch.envs.test.matrix]] +python = ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7","pypy3.8"] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:testing}" + + +[tool.hatch.envs.docs] +features = ["docs"] + +[tool.hatch.envs.docs.scripts] +html = "sphinx-build -W -b html doc doc/_build" + + +[tool.hatch.envs.pre-commit] +detached = true +dependences = ["pre-commit>1.11.0"] + +[tool.hatch.envs.pre-commit.scripts] +all = "pre-commit run --all-files --show-diff-on-failure" + [tool.hatch.build.hooks.vcs] -version-file = "execnet/_version.py" +version-file = "src/execnet/_version.py" -[tool.hatch.build.targets.sdist] -include = [ - "/execnet", -] +[tool.hatch.build] +sources = ["src"] [tool.mypy] python_version = "3.7" +mypy_path = "$MYPY_CONFIG_FILE_DIR/src" diff --git a/execnet/__init__.py b/src/execnet/__init__.py similarity index 96% rename from execnet/__init__.py rename to src/execnet/__init__.py index c403e2bb..8f7ddbe9 100644 --- a/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -7,6 +7,7 @@ (c) 2012, Holger Krekel and others """ from ._version import version as __version__ +from .gateway_base import Channel from .gateway_base import DataFormatError from .gateway_base import dump from .gateway_base import dumps diff --git a/execnet/gateway.py b/src/execnet/gateway.py similarity index 98% rename from execnet/gateway.py rename to src/execnet/gateway.py index 3d0bdddf..87cc9921 100644 --- a/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -170,10 +170,7 @@ def rinfo_source(channel): def _find_non_builtin_globals(source, codeobj): import ast - try: - import __builtin__ - except ImportError: - import builtins as __builtin__ + import builtins as __builtin__ vars = dict.fromkeys(codeobj.co_varnames) return [ diff --git a/execnet/gateway_base.py b/src/execnet/gateway_base.py similarity index 100% rename from execnet/gateway_base.py rename to src/execnet/gateway_base.py diff --git a/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py similarity index 100% rename from execnet/gateway_bootstrap.py rename to src/execnet/gateway_bootstrap.py diff --git a/execnet/gateway_io.py b/src/execnet/gateway_io.py similarity index 100% rename from execnet/gateway_io.py rename to src/execnet/gateway_io.py diff --git a/execnet/gateway_socket.py b/src/execnet/gateway_socket.py similarity index 100% rename from execnet/gateway_socket.py rename to src/execnet/gateway_socket.py diff --git a/execnet/multi.py b/src/execnet/multi.py similarity index 100% rename from execnet/multi.py rename to src/execnet/multi.py diff --git a/execnet/rsync.py b/src/execnet/rsync.py similarity index 100% rename from execnet/rsync.py rename to src/execnet/rsync.py diff --git a/execnet/rsync_remote.py b/src/execnet/rsync_remote.py similarity index 100% rename from execnet/rsync_remote.py rename to src/execnet/rsync_remote.py diff --git a/execnet/script/__init__.py b/src/execnet/script/__init__.py similarity index 100% rename from execnet/script/__init__.py rename to src/execnet/script/__init__.py diff --git a/execnet/script/loop_socketserver.py b/src/execnet/script/loop_socketserver.py similarity index 100% rename from execnet/script/loop_socketserver.py rename to src/execnet/script/loop_socketserver.py diff --git a/execnet/script/quitserver.py b/src/execnet/script/quitserver.py similarity index 100% rename from execnet/script/quitserver.py rename to src/execnet/script/quitserver.py diff --git a/execnet/script/shell.py b/src/execnet/script/shell.py similarity index 100% rename from execnet/script/shell.py rename to src/execnet/script/shell.py diff --git a/execnet/script/socketserver.py b/src/execnet/script/socketserver.py similarity index 100% rename from execnet/script/socketserver.py rename to src/execnet/script/socketserver.py diff --git a/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py similarity index 92% rename from execnet/script/socketserverservice.py rename to src/execnet/script/socketserverservice.py index c85f00fb..256c0df1 100644 --- a/execnet/script/socketserverservice.py +++ b/src/execnet/script/socketserverservice.py @@ -9,11 +9,11 @@ import sys import threading -import servicemanager -import win32event -import win32evtlogutil -import win32service -import win32serviceutil +import servicemanager # type: ignore[import] +import win32event # type: ignore[import] +import win32evtlogutil # type: ignore[import] +import win32service # type: ignore[import] +import win32serviceutil # type: ignore[import] appname = "ExecNetSocketServer" diff --git a/execnet/script/xx.py b/src/execnet/script/xx.py similarity index 61% rename from execnet/script/xx.py rename to src/execnet/script/xx.py index fd514c13..69f1ab52 100644 --- a/execnet/script/xx.py +++ b/src/execnet/script/xx.py @@ -1,7 +1,7 @@ import sys -import register -import rlcompleter2 +import register # type: ignore[import] +import rlcompleter2 # type: ignore[import] rlcompleter2.setup() diff --git a/execnet/xspec.py b/src/execnet/xspec.py similarity index 78% rename from execnet/xspec.py rename to src/execnet/xspec.py index 4d33ad67..39f0009d 100644 --- a/execnet/xspec.py +++ b/src/execnet/xspec.py @@ -1,6 +1,7 @@ """ (c) 2008-2013, holger krekel """ +from __future__ import annotations class XSpec: @@ -13,9 +14,20 @@ class XSpec: """ # XXX allow customization, for only allow specific key names - popen = ( - ssh - ) = socket = python = chdir = nice = dont_write_bytecode = execmodel = None + + _spec: str + + id: str | None = None + popen: str | bool | None = None + ssh: str | None = None + socket: str | bool | None = None + python: str | bool | None = None + chdir: str | bool | None = None + nice: str | bool | None = None + dont_write_bytecode: str | bool | None = None + execmodel: str | bool | None = None + + env: dict[str, str] def __init__(self, string): self._spec = string @@ -35,7 +47,7 @@ def __init__(self, string): else: setattr(self, key, value) - def __getattr__(self, name): + def __getattr__(self, name: str) -> str | bool | None: if name[0] == "_": raise AttributeError(name) return None diff --git a/testing/test_gateway.py b/testing/test_gateway.py index da498988..0ba01aa5 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -122,7 +122,7 @@ def remote(): ) monkeypatch.syspath_prepend(tmpdir) - import remote + import remote # type: ignore[import] ch = gw.remote_exec(remote) # simulate sending the code to a remote location that does not have @@ -151,7 +151,7 @@ def run_me(channel=None): ) monkeypatch.syspath_prepend(tmpdir) - import remotetest + import remotetest # type: ignore[import] ch = gw.remote_exec(remotetest) try: diff --git a/testing/test_serializer.py b/testing/test_serializer.py index bb032461..9f77c6e9 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -27,7 +27,7 @@ def _find_version(suffix=""): from pathlib import Path -# we use the execnet folder in order to avoid tiggering a missing apipkg +# we use the execnet folder in order to avoid triggering a missing apipkg pyimportdir = os.fspath(Path(execnet.__file__).parent) @@ -48,7 +48,7 @@ def dump(self, obj_rep: str) -> bytes: """ ) res = subprocess.run( - [self.executable, script_file], capture_output=True, check=True + [self.executable, os.fspath(script_file)], capture_output=True, check=True ) return res.stdout @@ -69,7 +69,7 @@ def load(self, data: bytes): """ ) res = subprocess.run( - [self.executable, script_file], + [self.executable, os.fspath(script_file)], capture_output=True, ) if res.returncode: diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b7a2db8a..00000000 --- a/tox.ini +++ /dev/null @@ -1,31 +0,0 @@ -[tox] -envlist=py{37,38,39,310,311,pypy37},docs,linting -isolated_build = true -[testenv] -usedevelop=true -deps= - pytest - pytest-timeout -passenv = GITHUB_ACTIONS HOME USER, XDG_* -commands= - python -m pytest {posargs:testing} - -[testenv:docs] -skipsdist = True -usedevelop = True -changedir = doc -deps = - sphinx - PyYAML -commands = - sphinx-build -W -b html . _build - -[testenv:linting] -skip_install = True -deps = pre-commit>=1.11.0 -commands = pre-commit run --all-files --show-diff-on-failure - -[pytest] -timeout = 20 -addopts = -ra -testpaths = testing From e51a0cc0b295a96145912e19c6446635c11dfbf8 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 11 Jan 2023 12:24:16 +0100 Subject: [PATCH 3/9] chore: unique list of found globals --- src/execnet/gateway.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 87cc9921..a8b3e7c7 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -172,14 +172,15 @@ def _find_non_builtin_globals(source, codeobj): import builtins as __builtin__ - vars = dict.fromkeys(codeobj.co_varnames) - return [ - node.id - for node in ast.walk(ast.parse(source)) - if isinstance(node, ast.Name) - and node.id not in vars - and node.id not in __builtin__.__dict__ - ] + vars = set(codeobj.co_varnames) + vars.update(__builtin__.__dict__) + + res = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Name) and node.id not in vars: + vars.add(node.id) + res.append(node.id) + return res def _source_of_function(function): From f2ce572af1117a414318be2a33bdc834d247c4d0 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 11 Jan 2023 12:24:46 +0100 Subject: [PATCH 4/9] workaround global finder bug when nested functions and imports are in play --- src/execnet/gateway.py | 3 ++- testing/test_basics.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index a8b3e7c7..c861e3fe 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -211,7 +211,8 @@ def _source_of_function(function): source = textwrap.dedent(source) # just for inner functions used_globals = _find_non_builtin_globals(source, codeobj) - if used_globals: + if used_globals and False: + # disabled this check as it fails for more complex examples raise ValueError("the use of non-builtin globals isn't supported", used_globals) leading_ws = "\n" * (codeobj.co_firstlineno - 1) diff --git a/testing/test_basics.py b/testing/test_basics.py index afc292bb..cd6539ec 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -397,6 +397,7 @@ def f(): assert self.check(f) == [] + @pytest.mark.xfail(reason="test disabled due to bugs") def test_function_with_global_fails(self): def func(channel): sys From e64547b82896dff4489b56d14a8c00ee3c6535dd Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 11 Jan 2023 07:54:33 +0100 Subject: [PATCH 5/9] implement rsync tests using pathlib.Path --- src/execnet/multi.py | 7 +- src/execnet/rsync.py | 89 +++++++++--------- src/execnet/rsync_remote.py | 9 +- testing/test_rsync.py | 182 ++++++++++++++++++++++-------------- 4 files changed, 162 insertions(+), 125 deletions(-) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 285851b5..ce38e456 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -12,6 +12,7 @@ from . import gateway_bootstrap from . import gateway_io +from .gateway_base import Channel from .gateway_base import get_execmodel from .gateway_base import reraise from .gateway_base import trace @@ -234,7 +235,7 @@ def remote_exec(self, source, **kwargs): class MultiChannel: - def __init__(self, channels): + def __init__(self, channels: list[Channel]): self._channels = channels def __len__(self): @@ -243,8 +244,8 @@ def __len__(self): def __iter__(self): return iter(self._channels) - def __getitem__(self, key): - return self._channels[key] + def __getitem__(self, index): + return self._channels[index] def __contains__(self, chan): return chan in self._channels diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index 32911e7e..8553c17a 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -3,12 +3,18 @@ (c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski """ +from __future__ import annotations + import os import stat from hashlib import md5 from queue import Queue +from typing import Callable +from typing import Type import execnet.rsync_remote +from execnet.gateway_base import Channel +from execnet.multi import MultiChannel class RSync: @@ -21,42 +27,45 @@ class RSync: a path on remote side). """ - def __init__(self, sourcedir, callback=None, verbose=True): + def __init__(self, sourcedir: str | os.PathLike[str], callback=None, verbose=True): self._sourcedir = str(sourcedir) self._verbose = verbose assert callback is None or hasattr(callback, "__call__") self._callback = callback - self._channels = {} - self._receivequeue = Queue() - self._links = [] + self._channels: dict[Channel, Callable[[], None] | None] = {} + self._links: list[tuple[str, str, str]] = [] def filter(self, path): return True - def _end_of_channel(self, channel): + def _end_of_channel(self, channel, data): if channel in self._channels: # too early! we must have got an error channel.waitclose() # or else we raise one raise OSError(f"connection unexpectedly closed: {channel.gateway} ") - def _process_link(self, channel): + def _process_link(self, channel, data): for link in self._links: channel.send(link) # completion marker, this host is done channel.send(42) - def _done(self, channel): + def _done(self, channel, data): """Call all callbacks""" finishedcallback = self._channels.pop(channel) if finishedcallback: finishedcallback() channel.waitclose() - def _list_done(self, channel): + def _ack(self, channel, data): + if self._callback: + self._callback("ack", self._paths[data], channel) + + def _list_done(self, channel, data): # sum up all to send if self._callback: - s = sum([self._paths[i] for i in self._to_send[channel]]) + s = sum(self._paths[i] for i in self._to_send[channel]) self._callback("list", s, channel) def _send_item(self, channel, data): @@ -64,8 +73,8 @@ def _send_item(self, channel, data): modified_rel_path, checksum = data modifiedpath = os.path.join(self._sourcedir, *modified_rel_path) try: - f = open(modifiedpath, "rb") - data = f.read() + with open(modifiedpath, "rb") as fp: + data = fp.read() except OSError: data = None @@ -81,7 +90,6 @@ def _send_item(self, channel, data): # print "sending", modified_rel_path, data and len(data) or 0, checksum if data is not None: - f.close() if checksum is not None and checksum == md5(data).digest(): data = None # not really modified else: @@ -92,7 +100,7 @@ def _report_send_file(self, gateway, modified_rel_path): if self._verbose: print(f"{gateway} <= {modified_rel_path}") - def send(self, raises=True): + def send(self, raises: bool = True) -> None: """Sends a sourcedir to all added targets. Flag indicates whether to raise an error or return in case of lack of targets @@ -110,45 +118,34 @@ def send(self, raises=True): # paths and to_send are only used for doing # progress-related callbacks - self._paths = {} - self._to_send = {} + self._paths: dict[str, int] = {} + self._to_send: dict[Channel, list[str]] = {} + + mch = MultiChannel(list(self._channels)) + rq = mch.make_receive_queue(endmarker=(None, None)) # send modified file to clients + commands: dict[str | None, Callable] = { + None: self._end_of_channel, + "links": self._process_link, + "done": self._done, + "ack": self._ack, + "send": self._send_item, + "list_done": self._list_done, + } + while self._channels: - channel, req = self._receivequeue.get() - if req is None: - self._end_of_channel(channel) - else: - command, data = req - if command == "links": - self._process_link(channel) - elif command == "done": - self._done(channel) - elif command == "ack": - if self._callback: - self._callback("ack", self._paths[data], channel) - elif command == "list_done": - self._list_done(channel) - elif command == "send": - self._send_item(channel, data) - del data - else: - assert "Unknown command %s" % command - - def add_target(self, gateway, destdir, finishedcallback=None, **options): + channel, (command, data) = rq.get() + assert command in commands, "Unknown command %s" % command + commands[command](channel, data) + + def add_target(self, gateway, destdir, finishedcallback=None, delete: bool = False): """Adds a remote target specified via a gateway and a remote destination directory. """ - for name in options: - assert name in ("delete",) - - def itemcallback(req): - self._receivequeue.put((channel, req)) - - channel = gateway.remote_exec(execnet.rsync_remote) - channel.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False) - channel.setcallback(itemcallback, endmarker=None) - channel.send((str(destdir), options)) + channel = gateway.remote_exec( + execnet.rsync_remote.serve_rsync, destdir=str(destdir), delete=delete + ) self._channels[channel] = finishedcallback def _broadcast(self, msg): diff --git a/src/execnet/rsync_remote.py b/src/execnet/rsync_remote.py index 4ac1880f..bdab406a 100644 --- a/src/execnet/rsync_remote.py +++ b/src/execnet/rsync_remote.py @@ -3,13 +3,12 @@ """ -def serve_rsync(channel): +def serve_rsync(channel, destdir: str, delete: bool = False): import os import stat import shutil from hashlib import md5 - destdir, options = channel.receive() modifiedfiles = [] def remove(path): @@ -44,7 +43,7 @@ def receive_directory_structure(path, relcomponents): destpath = os.path.join(path, entryname) receive_directory_structure(destpath, relcomponents + [entryname]) entrynames[entryname] = True - if options.get("delete"): + if delete: for othername in os.listdir(path): if othername not in entrynames: otherpath = os.path.join(path, othername) @@ -111,7 +110,3 @@ def receive_directory_structure(path, relcomponents): os.symlink(src, path) msg = channel.receive() channel.send(("done", None)) - - -if __name__ == "__channelexec__": - serve_rsync(channel) # type: ignore[name-defined] diff --git a/testing/test_rsync.py b/testing/test_rsync.py index a90f6654..dda6b37f 100644 --- a/testing/test_rsync.py +++ b/testing/test_rsync.py @@ -1,9 +1,27 @@ +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + import execnet import pytest from execnet import RSync from test_serializer import _find_version +def readlink(path: str | Path): + link = os.readlink(os.fspath(path)) + return Path(link) + + +skip_on_windows = pytest.mark.skipif( + sys.platform == "win32" or getattr(os, "_name", "") == "nt", + reason="broken on windows", +) + + @pytest.fixture(scope="module") def group(request): group = execnet.Group() @@ -25,16 +43,20 @@ def gw2(request, group): return gw -@pytest.fixture -def dirs(request, tmpdir): - t = tmpdir +class _dirs(argparse.Namespace): + source: Path + dest1: Path + dest2: Path - class dirs: - source = t.join("source") - dest1 = t.join("dest1") - dest2 = t.join("dest2") - return dirs +@pytest.fixture +def dirs(request, tmp_path) -> _dirs: + + return _dirs( + source=tmp_path / "source", + dest1=tmp_path / "dest1", + dest2=tmp_path / "dest2", + ) class TestRSync: @@ -48,57 +70,59 @@ def test_dirsync(self, dirs, gw1, gw2): dest = dirs.dest1 dest2 = dirs.dest2 source = dirs.source - + source.joinpath("subdir").mkdir(parents=True) for s in ("content1", "content2", "content2-a-bit-longer"): - source.ensure("subdir", "file1").write(s) + source.joinpath("subdir", "file1").write_text(s) rsync = RSync(dirs.source) rsync.add_target(gw1, dest) rsync.add_target(gw2, dest2) rsync.send() - assert dest.join("subdir").check(dir=1) - assert dest.join("subdir", "file1").check(file=1) - assert dest.join("subdir", "file1").read() == s - assert dest2.join("subdir").check(dir=1) - assert dest2.join("subdir", "file1").check(file=1) - assert dest2.join("subdir", "file1").read() == s + assert dest.joinpath("subdir").is_dir() + assert dest.joinpath("subdir", "file1").is_file() + assert dest.joinpath("subdir", "file1").read_text() == s + assert dest2.joinpath("subdir").is_dir() + assert dest2.joinpath("subdir", "file1").is_file() + assert dest2.joinpath("subdir", "file1").read_text() == s for x in dest, dest2: - fn = x.join("subdir", "file1") - fn.setmtime(0) + fn = x.joinpath("subdir", "file1") + os.utime(fn, (0, 0)) - source.join("subdir").remove("file1") + source.joinpath("subdir", "file1").unlink() rsync = RSync(source) rsync.add_target(gw2, dest2) rsync.add_target(gw1, dest) rsync.send() - assert dest.join("subdir", "file1").check(file=1) - assert dest2.join("subdir", "file1").check(file=1) + assert dest.joinpath("subdir", "file1").is_file() + assert dest2.joinpath("subdir", "file1").is_file() rsync = RSync(source) rsync.add_target(gw1, dest, delete=True) rsync.add_target(gw2, dest2) rsync.send() - assert not dest.join("subdir", "file1").check() - assert dest2.join("subdir", "file1").check() + assert not dest.joinpath("subdir", "file1").is_file() + assert dest2.joinpath("subdir", "file1").is_file() def test_dirsync_twice(self, dirs, gw1, gw2): source = dirs.source - source.ensure("hello") + source.mkdir() + source.joinpath("hello").touch() rsync = RSync(source) rsync.add_target(gw1, dirs.dest1) rsync.send() - assert dirs.dest1.join("hello").check() + assert dirs.dest1.joinpath("hello").is_file() with pytest.raises(IOError): rsync.send() assert rsync.send(raises=False) is None rsync.add_target(gw1, dirs.dest2) rsync.send() - assert dirs.dest2.join("hello").check() + assert dirs.dest2.joinpath("hello").is_file() with pytest.raises(IOError): rsync.send() assert rsync.send(raises=False) is None def test_rsync_default_reporting(self, capsys, dirs, gw1): source = dirs.source - source.ensure("hello") + source.mkdir() + source.joinpath("hello").touch() rsync = RSync(source) rsync.add_target(gw1, dirs.dest1) rsync.send() @@ -107,7 +131,8 @@ def test_rsync_default_reporting(self, capsys, dirs, gw1): def test_rsync_non_verbose(self, capsys, dirs, gw1): source = dirs.source - source.ensure("hello") + source.mkdir() + source.joinpath("hello").touch() rsync = RSync(source, verbose=False) rsync.add_target(gw1, dirs.dest1) rsync.send() @@ -115,47 +140,49 @@ def test_rsync_non_verbose(self, capsys, dirs, gw1): assert not out assert not err - @pytest.mark.skipif("sys.platform == 'win32' or getattr(os, '_name', '') == 'nt'") + @skip_on_windows def test_permissions(self, dirs, gw1, gw2): source = dirs.source dest = dirs.dest1 - onedir = dirs.source.ensure("one", dir=1) + onedir = dirs.source.joinpath("one") + onedir.mkdir(parents=True) onedir.chmod(448) - onefile = dirs.source.ensure("file") + onefile = dirs.source / "file" + onefile.touch() onefile.chmod(504) - onefile_mtime = onefile.stat().mtime + onefile_mtime = onefile.stat().st_mtime rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() - destdir = dirs.dest1.join(onedir.basename) - destfile = dirs.dest1.join(onefile.basename) - assert destfile.stat().mode & 511 == 504 - mode = destdir.stat().mode + destdir = dirs.dest1 / onedir.name + destfile = dirs.dest1 / onefile.name + assert destfile.stat().st_mode & 511 == 504 + mode = destdir.stat().st_mode assert mode & 511 == 448 # transfer again with changed permissions onedir.chmod(504) onefile.chmod(448) - onefile.setmtime(onefile_mtime) + os.utime(onefile, (-1, onefile_mtime)) rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() - mode = destfile.stat().mode + mode = destfile.stat().st_mode assert mode & 511 == 448, mode - mode = destdir.stat().mode + mode = destdir.stat().st_mode assert mode & 511 == 504 - @pytest.mark.skipif("sys.platform == 'win32' or getattr(os, '_name', '') == 'nt'") + @skip_on_windows def test_read_only_directories(self, dirs, gw1): source = dirs.source dest = dirs.dest1 - source.ensure("sub", "subsub", dir=True) - source.join("sub").chmod(0o500) - source.join("sub", "subsub").chmod(0o500) + source.joinpath("sub", "subsub").mkdir(parents=True) + source.joinpath("sub").chmod(0o500) + source.joinpath("sub", "subsub").chmod(0o500) # The destination directories should be created with the write # permission forced, to avoid raising an EACCES error. @@ -163,47 +190,63 @@ def test_read_only_directories(self, dirs, gw1): rsync.add_target(gw1, dest) rsync.send() - assert dest.join("sub").stat().mode & 0o700 - assert dest.join("sub").join("subsub").stat().mode & 0o700 + assert dest.joinpath("sub").stat().st_mode & 0o700 + assert dest.joinpath("sub", "subsub").stat().st_mode & 0o700 + @skip_on_windows def test_symlink_rsync(self, dirs, gw1): + source = dirs.source dest = dirs.dest1 - sourcefile = dirs.source.ensure("subdir", "existent") - source.join("rellink").mksymlinkto(sourcefile, absolute=0) - source.join("abslink").mksymlinkto(sourcefile) + + file_Path = Path("subdir", "existent") + + sourcefile = source.joinpath(file_Path) + sourcefile.parent.mkdir(parents=True) + sourcefile.touch() + + source.joinpath("rellink").symlink_to(file_Path) + source.joinpath("abslink").symlink_to(sourcefile) rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() - expected = dest.join(sourcefile.relto(dirs.source)) - assert dest.join("rellink").readlink() == "subdir/existent" - assert dest.join("abslink").readlink() == expected + expected = dest.joinpath(file_Path) + assert readlink(dest / "rellink") == file_Path + assert readlink(dest / "abslink") == expected + @skip_on_windows def test_symlink2_rsync(self, dirs, gw1): source = dirs.source dest = dirs.dest1 - subdir = dirs.source.ensure("subdir", dir=1) - sourcefile = subdir.ensure("somefile") - subdir.join("link1").mksymlinkto(subdir.join("link2"), absolute=0) - subdir.join("link2").mksymlinkto(sourcefile, absolute=1) - subdir.join("link3").mksymlinkto(source.dirpath(), absolute=1) + subdir = dirs.source.joinpath("subdir") + subdir.mkdir(parents=True) + sourcefile = subdir.joinpath("somefile") + sourcefile.touch() + subdir.joinpath("link1").symlink_to("link2") + subdir.joinpath("link2").symlink_to(sourcefile) + subdir.joinpath("link3").symlink_to( + source.parent, + target_is_directory=True, + ) rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() - expected = dest.join(sourcefile.relto(dirs.source)) - destsub = dest.join("subdir") - assert destsub.check() - assert destsub.join("link1").readlink() == "link2" - assert destsub.join("link2").readlink() == expected - assert destsub.join("link3").readlink() == source.dirpath() + expected = dest.joinpath(sourcefile.relative_to(dirs.source)) + destsub = dest.joinpath("subdir") + assert destsub.exists() + assert readlink(destsub / "link1") == Path("link2") + # resolve for windows quirk + assert readlink(destsub / "link2").resolve(strict=True) == expected + assert readlink(destsub / "link3") == source.parent def test_callback(self, dirs, gw1): dest = dirs.dest1 source = dirs.source - source.ensure("existent").write("a" * 100) - source.ensure("existant2").write("a" * 10) + source.mkdir() + source.joinpath("existent").write_text("a" * 100) + source.joinpath("existant2").write_text("a" * 10) total = {} def callback(cmd, lgt, channel): @@ -219,20 +262,21 @@ def callback(cmd, lgt, channel): def test_file_disappearing(self, dirs, gw1): dest = dirs.dest1 source = dirs.source - source.ensure("ex").write("a" * 100) - source.ensure("ex2").write("a" * 100) + source.mkdir() + source.joinpath("ex").write_text("a" * 100) + source.joinpath("ex2").write_text("a" * 100) class DRsync(RSync): def filter(self, x): assert x != source if x.endswith("ex2"): self.x = 1 - source.join("ex2").remove() + source.joinpath("ex2").unlink() return True rsync = DRsync(source) rsync.add_target(gw1, dest) rsync.send() assert rsync.x == 1 - assert len(dest.listdir()) == 1 - assert len(source.listdir()) == 1 + assert len(list(dest.iterdir())) == 1 + assert len(list(source.iterdir())) == 1 From 573554d6e16f3926af6d2129a39a59acac526928 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 12 Jan 2023 14:31:14 +0100 Subject: [PATCH 6/9] more windows hacks --- .github/workflows/main.yml | 2 +- src/execnet/rsync.py | 1 - testing/test_termination.py | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 14547db6..6851f89f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, osx-latest] + os: [windows-latest, ubuntu-latest] #, osx-latest] python: ["3.7","3.8","3.10","3.11", "pypy-3.7", "pypy-3.8"] steps: diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index 8553c17a..1466c9c5 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -124,7 +124,6 @@ def send(self, raises: bool = True) -> None: mch = MultiChannel(list(self._channels)) rq = mch.make_receive_queue(endmarker=(None, None)) - # send modified file to clients commands: dict[str | None, Callable] = { None: self._end_of_channel, "links": self._process_link, diff --git a/testing/test_termination.py b/testing/test_termination.py index ce496025..e63fdaa4 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -68,7 +68,8 @@ def test_termination_on_remote_channel_receive(monkeypatch, makegateway): gw._group.terminate() command = ["ps", "-p", str(pid)] output = subprocess.run(command, capture_output=True, text=True) - assert str(pid) not in output.stdout, output + print(output.stdout) + assert str(pid) not in output.stdout def test_close_initiating_remote_no_error(testdir, anypython): From a935b6b22621dcb2451787fcab52138218ba7f57 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 14 Jan 2023 13:34:31 +0100 Subject: [PATCH 7/9] state --- .pre-commit-config.yaml | 5 ++--- pyproject.toml | 2 +- src/execnet/gateway_base.py | 15 ++------------- src/execnet/gateway_bootstrap.py | 12 +++++++----- src/execnet/gateway_io.py | 2 +- src/execnet/multi.py | 1 - testing/conftest.py | 2 +- testing/test_basics.py | 3 ++- 8 files changed, 16 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00eb6421..1f813277 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,3 @@ - repos: - repo: https://github.com/codespell-project/codespell rev: v2.2.2 @@ -23,12 +22,12 @@ repos: rev: v3.3.1 hooks: - id: pyupgrade - args: [--py37-plus] + args: [--py38-plus] - repo: https://github.com/asottile/reorder_python_imports rev: v3.9.0 hooks: - id: reorder-python-imports - args: ['--application-directories=execnet', --py37-plus, "--remove-import=import py"] + args: ['--application-directories=execnet', --py38-plus, "--remove-import=import py"] - repo: https://github.com/PyCQA/doc8 rev: 'v1.1.1' hooks: diff --git a/pyproject.toml b/pyproject.toml index 9403e1a6..92c93797 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,5 +93,5 @@ sources = ["src"] [tool.mypy] -python_version = "3.7" +python_version = "3.8" mypy_path = "$MYPY_CONFIG_FILE_DIR/src" diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 9577a94b..11962db7 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -18,20 +18,10 @@ import sys import traceback import weakref +from _thread import interrupt_main from io import BytesIO from typing import Callable -from typing import IO - - -def reraise(cls, val, tb): - raise val.with_traceback(tb) - - -unicode = str -_long_type = int -from _thread import interrupt_main -SUBPROCESS32 = False # f = open("/tmp/execnet-%s" % os.getpid(), "w") # def log_extra(*msg): # f.write(" ".join([str(x) for x in msg]) + "\n") @@ -46,9 +36,8 @@ def get_execmodel(backend): return backend if backend == "thread": importdef = { - "get_ident": ["thread::get_ident", "_thread::get_ident"], + "get_ident": ["_thread::get_ident"], "_start_new_thread": [ - "thread::start_new_thread", "_thread::start_new_thread", ], "threading": ["threading"], diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py index 30b6d866..9df1a3dc 100644 --- a/src/execnet/gateway_bootstrap.py +++ b/src/execnet/gateway_bootstrap.py @@ -2,6 +2,7 @@ code to initialize the remote side of a gateway once the io is created """ import inspect +import json import os import execnet @@ -23,13 +24,13 @@ def bootstrap_import(io, spec): sendexec( io, "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, + f"if {importdir!r} not in sys.path:", + f" sys.path.insert(0, {importdir!r})", "from execnet.gateway_base import serve, init_popen_io, get_execmodel", "sys.stdout.write('1')", "sys.stdout.flush()", - "execmodel = get_execmodel(%r)" % spec.execmodel, - "serve(init_popen_io(execmodel), id='%s-worker')" % spec.id, + f"execmodel = get_execmodel({spec.execmodel!r})", + f"serve(init_popen_io(execmodel), id='{spec.id}-worker')", ) s = io.read(1) assert s == b"1", repr(s) @@ -75,7 +76,8 @@ def bootstrap_socket(io, id): def sendexec(io, *sources): source = "\n".join(sources) - io.write((repr(source) + "\n").encode("ascii")) + encoded = (json.dumps(source) + "\n").encode("ascii") + io.write(encoded) def fix_pid_for_jython_popen(gw): diff --git a/src/execnet/gateway_io.py b/src/execnet/gateway_io.py index 84352655..4b2a7279 100644 --- a/src/execnet/gateway_io.py +++ b/src/execnet/gateway_io.py @@ -38,7 +38,7 @@ def killpopen(popen): sys.stderr.flush() -popen_bootstrapline = "import sys;exec(eval(sys.stdin.readline()))" +popen_bootstrapline = "import sys;import json;exec(json.loads(sys.stdin.readline()))" def shell_split_path(path): diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ce38e456..1e96dfa3 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -14,7 +14,6 @@ from . import gateway_io from .gateway_base import Channel from .gateway_base import get_execmodel -from .gateway_base import reraise from .gateway_base import trace from .xspec import XSpec diff --git a/testing/conftest.py b/testing/conftest.py index 6b472576..05d1e414 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -109,7 +109,7 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("gw", gwtypes, indirect=True) -@lru_cache() +@lru_cache def getexecutable(name): if name == "sys.executable": return sys.executable diff --git a/testing/test_basics.py b/testing/test_basics.py index cd6539ec..9bafc45c 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -1,6 +1,7 @@ from __future__ import annotations import inspect +import json import os import subprocess import sys @@ -86,7 +87,7 @@ def receive(): try: source = f"{inspect.getsource(read_write_loop)}\n\nread_write_loop()" - repr_source = repr(str(source)) + "\n" + repr_source = json.dumps(source) + "\n" sendline = repr_source send(sendline) s = receive() From 22e8ffbfefa655a1c24fdebdc8e54c939fd4e97b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 15 Jan 2023 22:26:14 +0100 Subject: [PATCH 8/9] drop python 3.7 --- .github/workflows/main.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6851f89f..a80851a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-latest] #, osx-latest] - python: ["3.7","3.8","3.10","3.11", "pypy-3.7", "pypy-3.8"] + python: ["3.8","3.10","3.11", "pypy-3.8"] steps: - uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index 92c93797..9d85fc47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ source = "vcs" features = ["testing"] [[tool.hatch.envs.test.matrix]] -python = ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7","pypy3.8"] +python = ["3.8", "3.9", "3.10", "3.11", "pypy3.8"] [tool.hatch.envs.test.scripts] test = "pytest {args:testing}" From 6c0654a5bc063fb10316aee6a395b9855cd48fad Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Mon, 16 Jan 2023 00:35:37 +0100 Subject: [PATCH 9/9] fixups ball of mud --- pyproject.toml | 2 +- src/execnet/gateway_base.py | 21 ++++++++++++--------- testing/test_xspec.py | 9 ++------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9d85fc47..90c0b9d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ html = "sphinx-build -W -b html doc doc/_build" [tool.hatch.envs.pre-commit] detached = true -dependences = ["pre-commit>1.11.0"] +dependences = ["pre-commit>=2.20.0"] [tool.hatch.envs.pre-commit.scripts] all = "pre-commit run --all-files --show-diff-on-failure" diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 11962db7..a53e5ec5 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1,7 +1,7 @@ """ base execnet gateway code send to the other side for bootstrapping. -NOTE: aims to be compatible to Python 2.5-3.X, Jython and IronPython +NOTE: aims to be compatible to Python 3.8+ :copyright: 2004-2015 :authors: @@ -153,6 +153,8 @@ class Reply: through WorkerPool.spawn() """ + _exception: BaseException | None = None + def __init__(self, task, threadmodel): self.task = task self._result_ready = threadmodel.Event() @@ -165,10 +167,10 @@ def get(self, timeout=None): including its traceback. """ self.waitfinish(timeout) - try: + if self._exception is None: return self._result - except AttributeError: - reraise(*(self._excinfo[:3])) # noqa + else: + raise self._exception.with_traceback(self._exception.__traceback__) def waitfinish(self, timeout=None): if not self._result_ready.wait(timeout): @@ -179,10 +181,9 @@ def run(self): try: try: self._result = func(*args, **kwargs) - except: + except BaseException as e: # sys may be already None when shutting down the interpreter - if sys is not None: - self._excinfo = sys.exc_info() + self._exception = e finally: self._result_ready.set() self.running = False @@ -348,7 +349,9 @@ def __init__(self, outfile, infile, execmodel): except (AttributeError, OSError): pass self._read = getattr(infile, "buffer", infile).read - self._write = getattr(outfile, "buffer", outfile).write + _outfile = getattr(outfile, "buffer", outfile) + self._write = _outfile.write + self._flush = _outfile.flush self.execmodel = execmodel def read(self, numbytes): @@ -366,7 +369,7 @@ def write(self, data): """write out all data bytes.""" assert isinstance(data, bytes) self._write(data) - self.outfile.flush() + self._flush() def close_read(self): self.infile.close() diff --git a/testing/test_xspec.py b/testing/test_xspec.py index a841dfb6..64756a58 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -8,6 +8,7 @@ import execnet import pytest from execnet.gateway_io import popen_args +from execnet.gateway_io import popen_bootstrapline from execnet.gateway_io import ssh_args from execnet.gateway_io import vagrant_ssh_args @@ -77,13 +78,7 @@ def test_vagrant_options(self): def test_popen_with_sudo_python(self): spec = XSpec("popen//python=sudo python3") - assert popen_args(spec) == [ - "sudo", - "python3", - "-u", - "-c", - "import sys;exec(eval(sys.stdin.readline()))", - ] + assert popen_args(spec) == ["sudo", "python3", "-u", "-c", popen_bootstrapline] def test_env(self): xspec = XSpec("popen//env:NAME=value1")