diff --git a/.gitignore b/.gitignore index b8f4031..d2d1f3e 100644 --- a/.gitignore +++ b/.gitignore @@ -234,3 +234,4 @@ Temporary Items .apdisk /poetry.lock +profile.json diff --git a/README.rst b/README.rst index b2ecbc0..f138b9a 100644 --- a/README.rst +++ b/README.rst @@ -12,6 +12,8 @@ Belay is a library that enables the rapid development of projects that interact Belay works by interacting with the REPL interface of a micropython board from Python code running on PC. +Belay supports wired serial connections (USB) and wireless connections via WebREPL over WiFi. + `Quick Video of Belay in 22 seconds.`_ See `the documentation`_ for usage and other details. diff --git a/belay/device.py b/belay/device.py index cba9aa2..64ce470 100644 --- a/belay/device.py +++ b/belay/device.py @@ -1,5 +1,4 @@ import ast -import binascii import hashlib import importlib.resources as pkg_resources import linecache @@ -13,6 +12,7 @@ from ._minify import minify as minify_code from .inspect import getsource from .pyboard import Pyboard, PyboardException +from .webrepl import WebreplToSerial # Typing PythonLiteral = Union[None, bool, bytes, int, float, str, List, Dict, Set] @@ -42,7 +42,7 @@ def _local_hash_file(fn): if not data: break hasher.update(data) - return binascii.hexlify(hasher.digest()).decode() + return hasher.digest() class _Executer(ABC): @@ -217,32 +217,30 @@ def __init__( Code to run on startup. Defaults to a few common imports. """ self._board = Pyboard(*args, **kwargs) - self._board.enter_raw_repl() + if isinstance(self._board.serial, WebreplToSerial): + soft_reset = False + else: + soft_reset = True + self._board.enter_raw_repl(soft_reset=soft_reset) self.task = _TaskExecuter(self) self.thread = _ThreadExecuter(self) - self._exec_snippet("startup") - if startup is None: - self._exec_snippet("convenience_imports") + self._exec_snippet("startup", "convenience_imports") elif startup: - self(startup) + self(_read_snippet("startup") + "\n" + startup) - def _exec_snippet(self, name: str, *args): + def _exec_snippet(self, *names: str): """Load and execute a snippet from the snippets sub-package. Parameters ---------- - name : str - Snippet to load. - args - If provided, substitutes into loaded snippet. + names : str + Snippet(s) to load and execute. """ - snippet = _read_snippet(name) - if args: - snippet = snippet % args - return self(snippet) + snippets = [_read_snippet(name) for name in names] + return self("\n".join(snippets)) def __call__( self, @@ -321,22 +319,35 @@ def sync( keep = ["boot.py", "webrepl_cfg.py"] elif isinstance(keep, str): keep = [keep] - for dst in keep: - if dst[0] != "/": - dst = "/" + dst - self(f'all_files.discard("{dst}")') + keep = [x if x[0] == "/" else "/" + x for x in keep] # Sort so that folder creation comes before file sending. - local_files = sorted(folder.rglob("*")) - for src in local_files: - dst = f"/{src.relative_to(folder)}" + src_objects = sorted(folder.rglob("*")) + src_files, src_dirs = [], [] + for src_object in src_objects: + if src_object.is_dir(): + src_dirs.append(src_object) + else: + src_files.append(src_object) + dst_files = [f"/{src.relative_to(folder)}" for src in src_files] + dst_dirs = [f"/{src.relative_to(folder)}" for src in src_dirs] + keep = [x for x in keep if x not in dst_files] + if dst_files + keep: + self(f"for x in {repr(dst_files + keep)}:\n all_files.discard(x)") - with tempfile.TemporaryDirectory() as tmp_dir: - tmp_dir = Path(tmp_dir) # Used if we need to perform a conversion + # Try and make all remote dirs + if dst_dirs: + self(f"__belay_mkdirs({repr(dst_dirs)})") - if src.is_dir(): - self._exec_snippet("try_mkdir", dst) - continue + # Get all remote hashes + dst_hashes = self(f"__belay_hfs({repr(dst_files)})") + + if len(dst_hashes) != len(dst_files): + raise Exception + + for src, dst, dst_hash in zip(src_files, dst_files, dst_hashes): + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir = Path(tmp_dir) if minify and src.suffix == ".py": minified = minify_code(src.read_text()) @@ -344,11 +355,9 @@ def sync( src.write_text(minified) # All other files, just sync over. - local_hash = _local_hash_file(src) - remote_hash = self(f"__belay_hash_file({repr(dst)})") - if local_hash != remote_hash: + src_hash = _local_hash_file(src) + if src_hash != dst_hash: self._board.fs_put(src, dst) - self(f'all_files.discard("{dst}")') # Remove all the files and directories that did not exist in local filesystem. self._exec_snippet("sync_end") diff --git a/belay/pyboard.py b/belay/pyboard.py index 4613946..09d7a95 100644 --- a/belay/pyboard.py +++ b/belay/pyboard.py @@ -72,6 +72,8 @@ import sys import time +from .webrepl import WebreplToSerial + try: stdout = sys.stdout.buffer except AttributeError: @@ -286,6 +288,8 @@ def __init__( ): # device looks like an IP address self.serial = TelnetToSerial(device, user, password, read_timeout=10) + elif device and device.startswith("ws://"): + self.serial = WebreplToSerial(device, password, read_timeout=10) else: import serial diff --git a/belay/snippets/sync_begin.py b/belay/snippets/sync_begin.py index dd4f3f5..93a64ff 100644 --- a/belay/snippets/sync_begin.py +++ b/belay/snippets/sync_begin.py @@ -1,26 +1,35 @@ # Creates and populates two set[str]: all_files, all_dirs -import os, hashlib, binascii -def __belay_hash_file(fn): - hasher = hashlib.sha256() +import os, hashlib, errno +def __belay_hf(fn): + h = hashlib.sha256() try: with open(fn, "rb") as f: while True: data = f.read(4096) if not data: break - hasher.update(data) + h.update(data) except OSError: - return "0" * 64 - return str(binascii.hexlify(hasher.digest())) + return b"" + return h.digest() +def __belay_hfs(fns): + print(repr([__belay_hf(fn) for fn in fns])) +def __belay_mkdirs(fns): + for fn in fns: + try: + os.mkdir('%s') + except OSError as e: + if e.errno != errno.EEXIST: + raise all_files, all_dirs = set(), [] -def enumerate_fs(path=""): +def __belay_fs(path=""): for elem in os.ilistdir(path): full_name = path + "/" + elem[0] if elem[1] & 0x4000: # is_dir all_dirs.append(full_name) - enumerate_fs(full_name) + __belay_fs(full_name) else: all_files.add(full_name) -enumerate_fs() +__belay_fs() all_dirs.sort() -del enumerate_fs +del __belay_fs diff --git a/belay/snippets/sync_end.py b/belay/snippets/sync_end.py index a7e5566..008fe00 100644 --- a/belay/snippets/sync_end.py +++ b/belay/snippets/sync_end.py @@ -5,4 +5,4 @@ os.rmdir(folder) except OSError: pass -del all_files, all_dirs, __belay_hash_file +del all_files, all_dirs, __belay_hf, __belay_hfs, __belay_mkdirs diff --git a/belay/snippets/try_mkdir.py b/belay/snippets/try_mkdir.py deleted file mode 100644 index c827a06..0000000 --- a/belay/snippets/try_mkdir.py +++ /dev/null @@ -1,6 +0,0 @@ -import os, errno -try: - os.mkdir('%s') -except OSError as e: - if e.errno != errno.EEXIST: - raise diff --git a/belay/webrepl.py b/belay/webrepl.py new file mode 100644 index 0000000..477b63f --- /dev/null +++ b/belay/webrepl.py @@ -0,0 +1,341 @@ +""" +The MIT License (MIT). + +Copyright (c) 2016 Damien P. George +Copyright (c) 2016 Paul Sokolovsky +Copyright (c) 2022 Jim Mussared + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import errno +import os +import socket +import struct +import sys +from collections import deque +from pathlib import Path + +# Treat this remote directory as a root for file transfers +SANDBOX = "" +# SANDBOX = "/tmp/webrepl/" +DEBUG = 0 + +WEBREPL_REQ_S = "<2sBBQLH64s" +WEBREPL_PUT_FILE = 1 +WEBREPL_GET_FILE = 2 +WEBREPL_GET_VER = 3 + + +def debugmsg(msg): + if DEBUG: + print(msg) + + +class WebreplError(Exception): + pass + + +class Websocket: + def __init__(self, s: socket.socket): + self.s = s + self.buf = b"" + + def write(self, data): + data_len = len(data) + if data_len < 126: + # TODO: hardcoded "binary" type + hdr = struct.pack(">BB", 0x82, data_len) + else: + hdr = struct.pack(">BBH", 0x82, 126, data_len) + self.s.send(hdr) + self.s.send(data) + + def writetext(self, data: bytes): + data_len = len(data) + if data_len < 126: + hdr = struct.pack(">BB", 0x81, data_len) + else: + hdr = struct.pack(">BBH", 0x81, 126, data_len) + self.s.send(hdr) + self.s.send(data) + + def recvexactly(self, sz): + res = b"" + while sz: + data = self.s.recv(sz) + if not data: + break + res += data + sz -= len(data) + return res + + def read(self, size, text_ok=False, size_match=True): + if not self.buf: + while True: + hdr = self.recvexactly(2) + if len(hdr) != 2: + raise WebreplError + fl, sz = struct.unpack(">BB", hdr) + if sz == 126: + hdr = self.recvexactly(2) + if len(hdr) != 2: + raise WebreplError + (sz,) = struct.unpack(">H", hdr) + if fl == 0x82: + break + if text_ok and fl == 0x81: + break + debugmsg("Got unexpected websocket record of type %x, skipping it" % fl) + while sz: + skip = self.s.recv(sz) + debugmsg("Skip data: %s" % skip) + sz -= len(skip) + data = self.recvexactly(sz) + if len(data) != sz: + raise WebreplError + self.buf = data + + d = self.buf[:size] + self.buf = self.buf[size:] + if size_match and len(d) != size: + raise WebreplError + return d + + def ioctl(self, req, val): + if req != 9 and val != 2: + raise WebreplError + + +def login(ws, passwd): + while True: + c = ws.read(1, text_ok=True) + if c == b":": + if ws.read(1, text_ok=True) != b" ": + raise WebreplError + break + ws.write(passwd.encode("utf-8") + b"\r") + + +def read_resp(ws): + data = ws.read(4) + sig, code = struct.unpack("<2sH", data) + if sig != b"WB": + raise WebreplError + return code + + +def send_req(ws, op, sz=0, fname=b""): + rec = struct.pack(WEBREPL_REQ_S, b"WA", op, 0, 0, sz, len(fname), fname) + debugmsg("%r %d" % (rec, len(rec))) + ws.write(rec) + + +def get_ver(ws): + send_req(ws, WEBREPL_GET_VER) + d = ws.read(3) + d = struct.unpack(": - Copy remote file to local file" + ) + print( + " [-p password] : - Copy local file to remote file" + ) + print("Examples:") + print(" %s script.py 192.168.4.1:/another_name.py" % exename) + print(" %s script.py 192.168.4.1:/app/" % exename) + print(" %s -p password 192.168.4.1:/app/script.py ." % exename) + sys.exit(rc) + + +def error(msg): + print(msg) + sys.exit(1) + + +def parse_remote(remote): + host, fname = remote.rsplit(":", 1) + if fname == "": + fname = "/" + port = 8266 + if ":" in host: + host, port = host.split(":") + port = int(port) + return (host, port, fname) + + +def client_handshake(sock): + """Simplified client handshake. + + Works for MicroPython's websocket server implementation, but probably not + for other servers. + """ + cl = sock.makefile("rwb", 0) + cl.write( + b"""\ +GET / HTTP/1.1\r +Host: echo.websocket.org\r +Connection: Upgrade\r +Upgrade: websocket\r +Sec-WebSocket-Key: foo\r +\r +""" + ) + line = cl.readline() + while 1: + line = cl.readline() + if line == b"\r\n": + break + + +class WebsocketClosedError(Exception): + """Attempted to use a closed websocket.""" + + +class WebreplToSerial: + def __init__(self, uri, password, read_timeout=None): + self.fifo = deque() + self.read_timeout = read_timeout + + if uri.startswith("ws://"): + uri = uri[5:] + host, *remain = uri.split(":", 1) + port = int(remain[0]) if remain else 8266 + + self.s = socket.socket() + self.s.settimeout(read_timeout) + self.s.connect((host, port)) + client_handshake(self.s) + + self.ws = Websocket(self.s) + + login(self.ws, password) + if self.read(1024) != b"\r\nWebREPL connected\r\n>>> ": + raise WebreplError + + def close(self): + if self.s is not None: + self.s.close() + self.s = self.ws = None + + def write(self, data: bytes) -> int: + if self.ws is None: + raise WebsocketClosedError + self.ws.writetext(data) + return len(data) + + def read(self, size=1) -> bytes: + if self.ws is None: + raise WebsocketClosedError + + readin = self.ws.read(size, text_ok=True, size_match=False) + self.fifo.extend(readin) + + data = b"" + while len(data) < size and len(self.fifo) > 0: + data += bytes([self.fifo.popleft()]) + return data + + def inWaiting(self): + if self.s is None or self.ws is None: + raise WebsocketClosedError + + n_waiting = len(self.fifo) + len(self.ws.buf) + if not n_waiting: + self.s.setblocking(False) + try: + n_waiting = len(self.s.recv(1024, socket.MSG_PEEK)) + except BlockingIOError: + pass + except socket.error as e: + if e == errno.EAGAIN or e == errno.EWOULDBLOCK: + pass + else: + raise + self.s.setblocking(True) + return n_waiting + else: + return n_waiting diff --git a/docs/source/Connections.rst b/docs/source/Connections.rst new file mode 100644 index 0000000..df01345 --- /dev/null +++ b/docs/source/Connections.rst @@ -0,0 +1,69 @@ +Connections +=========== + +Belay currently supports two connection mediums: + +1. Serial, typically over a USB cable. Recommended connection method. + +2. WebREPL, typically over WiFi. Experimental and relatively slow due to higher command latency. + + +Serial +^^^^^^ +This is the typical connection method over a cable and is fairly self-explanatory. + + +WebREPL +^^^^^^^ +WebREPL_ is a protocol for accessing a MicroPython REPL over websockets. + +WebREPL requires the MicroPython-bundled ``webrepl`` server running on-device. +To run the WebREPL server on boot, we need two files on device: + +1. ``boot.py`` that connects to your WiFi and starts the server. +2. ``webrepl_cfg.py`` that contains the password to access the WebREPL interface. + +These files may look like (tested on an ESP32): + +.. code-block:: python + + ########### + # boot.py # + ########### + def do_connect(ssid, pwd): + import network + + sta_if = network.WLAN(network.STA_IF) + if not sta_if.isconnected(): + print("connecting to network...") + sta_if.active(True) + sta_if.connect(ssid, pwd) + while not sta_if.isconnected(): + pass + print("network config:", sta_if.ifconfig()) + + + # Attempt to connect to WiFi network + do_connect("YOUR WIFI SSID", "YOUR WIFI PASSWORD") + + import webrepl + + webrepl.start() + +.. code-block:: python + + ################## + # webrepl_cfg.py # + ################## + PASS = "python" + +Once these files are on-device, connect to the device by providing the +correct IP address and password. The ``ws://`` prefix tells Belay to +use WebREPL. + +.. code-block:: python + + device = belay.Device("ws://192.168.1.100", password="python") + + +.. _WebREPL: https://github.com/micropython/webrepl diff --git a/docs/source/index.rst b/docs/source/index.rst index 23e4891..25b2df4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ Welcome to Belay's documentation! Installation Quick Start + Connections How Belay Works api diff --git a/examples/09_webrepl/README.rst b/examples/09_webrepl/README.rst new file mode 100644 index 0000000..8ceee21 --- /dev/null +++ b/examples/09_webrepl/README.rst @@ -0,0 +1,38 @@ +Example 09: WebREPL +=================== + +First, run the code using the serial port so that the configurations in the ``board/`` folder are synced to device. + +.. code-block: bash + + python main.py --port /dev/ttyUSB0 + +You can then get the IP address by examining the terminal output (using a tool like ``minicom`` or ``mpremote``) on boot, or looking at your router configuration. + +.. code-block: bash + + mpremote connect /dev/ttyUSB0 + +This should return something like: + +.. code-block: text + + Connected to MicroPython at /dev/ttyUSB0 + Use Ctrl-] to exit this shell + OK + MPY: soft reboot + connecting to network... + network config: ('192.168.1.110', '255.255.255.0', '192.168.1.1', '192.168.1.1') + WebREPL daemon started on ws://192.168.1.110:8266 + Started webrepl in normal mode + raw REPL; CTRL-B to exit + > + +Now that a WebREPL server is running on-device, and we know the device's IP address, we can wirelessly run the script via: + +.. code-block: bash + + python main.py --port ws://192.168.1.110 + + +https://micropython.org/webrepl/ diff --git a/examples/09_webrepl/board/boot.py b/examples/09_webrepl/board/boot.py new file mode 100644 index 0000000..b70c941 --- /dev/null +++ b/examples/09_webrepl/board/boot.py @@ -0,0 +1,19 @@ +def do_connect(ssid, pwd): + import network + + sta_if = network.WLAN(network.STA_IF) + if not sta_if.isconnected(): + print("connecting to network...") + sta_if.active(True) + sta_if.connect(ssid, pwd) + while not sta_if.isconnected(): + pass + print("network config:", sta_if.ifconfig()) + + +# Attempt to connect to WiFi network +do_connect("your wifi ssid", "your password") + +import webrepl + +webrepl.start() diff --git a/examples/09_webrepl/board/webrepl_cfg.py b/examples/09_webrepl/board/webrepl_cfg.py new file mode 100644 index 0000000..e01dee6 --- /dev/null +++ b/examples/09_webrepl/board/webrepl_cfg.py @@ -0,0 +1 @@ +PASS = "python" # nosec diff --git a/examples/09_webrepl/main.py b/examples/09_webrepl/main.py new file mode 100644 index 0000000..dbf4c55 --- /dev/null +++ b/examples/09_webrepl/main.py @@ -0,0 +1,38 @@ +import argparse +from time import sleep + +import belay + +parser = argparse.ArgumentParser() +parser.add_argument("--port", "-p", default="ws://192.168.1.100") +parser.add_argument("--password", default="python") +args = parser.parse_args() + +print("Connecting to device") +device = belay.Device(args.port, password=args.password) + +print("Syncing filesystem.") +# Sync our WiFi information and WebREPL configuration. +device.sync("board/") + + +print("Sending set_led task") + + +@device.task +def set_led(counter, state): + # Configuration for a Pi Pico board. + Pin(25, Pin.OUT).value(state) + return counter + + +for counter in range(10_000): + print("led on ", end="") + res = set_led(counter, True) + print(f"Counter: {res}", end="\r") + sleep(0.5) + + print("led off ", end="") + res = set_led(counter, False) + print(f"Counter: {res}", end="\r") + sleep(0.5) diff --git a/tests/test_device.py b/tests/test_device.py index 539c7d5..c5ebc12 100644 --- a/tests/test_device.py +++ b/tests/test_device.py @@ -7,7 +7,10 @@ @pytest.fixture def mock_pyboard(mocker): - mocker.patch("belay.device.Pyboard.__init__", return_value=None) + def mock_init(self, *args, **kwargs): + self.serial = None + + mocker.patch.object(belay.device.Pyboard, "__init__", mock_init) mocker.patch("belay.device.Pyboard.enter_raw_repl", return_value=None) mocker.patch("belay.device.Pyboard.exec", return_value=b"None") mocker.patch("belay.device.Pyboard.fs_put") @@ -135,26 +138,23 @@ def sync_path(tmp_path): def test_device_sync_empty_remote(mocker, mock_device, sync_path): - payload = bytes(repr("0" * 64), encoding="utf8") + payload = repr([b"\x00"] * 5).encode("utf-8") + # payload = [bytes(repr("0" * 64), encoding="utf8")] * 5 mock_device._board.exec = mocker.MagicMock(return_value=payload) mock_device.sync(sync_path) - expected_cmds = [ - "__belay_hash_file('/alpha.py')", - "__belay_hash_file('/bar.txt')", - "__belay_hash_file('/folder1/file1.txt')", - "__belay_hash_file('/folder1/folder1_1/file1_1.txt')", - "__belay_hash_file('/foo.txt')", - ] - call_args_list = mock_device._board.exec.call_args_list[1:] - assert len(expected_cmds) <= len(call_args_list) - for i, expected_cmd in enumerate(expected_cmds): - for actual_call in call_args_list: - if actual_call.args[-1] == expected_cmd: - break - else: - raise Exception(f"cmd {i} not found: {expected_cmd}") + mock_device._board.exec.assert_has_calls( + [ + call( + "for x in['/alpha.py','/bar.txt','/folder1/file1.txt','/folder1/folder1_1/file1_1.txt','/foo.txt','/boot.py','/webrepl_cfg.py']:\n all_files.discard(x)" + ), + call("__belay_mkdirs(['/folder1','/folder1/folder1_1'])"), + call( + "__belay_hfs(['/alpha.py','/bar.txt','/folder1/file1.txt','/folder1/folder1_1/file1_1.txt','/foo.txt'])" + ), + ] + ) mock_device._board.fs_put.assert_has_calls( [ @@ -170,18 +170,23 @@ def test_device_sync_empty_remote(mocker, mock_device, sync_path): def test_device_sync_partial_remote(mocker, mock_device, sync_path): - def __belay_hash_file(fn): - local_fn = sync_path / fn[1:] - if local_fn.stem.endswith("1"): - return "0" * 64 - else: - return belay.device._local_hash_file(local_fn) - - def side_effect(src_file, src_lineno, name, cmd): - nonlocal __belay_hash_file - return eval(cmd) - - mock_device._traceback_execute = mocker.MagicMock(side_effect=side_effect) + def __belay_hfs(fns): + out = [] + for fn in fns: + local_fn = sync_path / fn[1:] + if local_fn.stem.endswith("1"): + out.append(b"\x00") + else: + out.append(belay.device._local_hash_file(local_fn)) + return out + + def side_effect(cmd): + if not cmd.startswith("__belay_hfs"): + return b"" + nonlocal __belay_hfs + return repr(eval(cmd)).encode("utf-8") + + mock_device._board.exec = mocker.MagicMock(side_effect=side_effect) mock_device.sync(sync_path)