From f87cdf7f19d614d6f727faffa2dcec9ddc5ee22a Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Sat, 13 Aug 2022 11:31:07 -0700 Subject: [PATCH] change belay prefix to _belay_, change json_decorator to __belay_json, reduce whitespace in json data, rename hash_remote_file to __belay_hash_file to reduce chance of conflict --- belay/device.py | 40 ++++++++++++++++----------------- docs/source/How Belay Works.rst | 6 ++--- tests/test_device.py | 38 +++++++++++++++++-------------- 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/belay/device.py b/belay/device.py index 218a821..6835f5c 100644 --- a/belay/device.py +++ b/belay/device.py @@ -16,13 +16,13 @@ JsonSerializeable = Union[None, bool, int, float, str, List, Dict] # MicroPython Code Snippets -_BELAY_PREFIX = "__belay_" +_BELAY_PREFIX = "_belay_" -_BELAY_STARTUP_CODE = f"""import ujson -def json_decorator(f): +_BELAY_STARTUP_CODE = f"""import json +def __belay_json(f): def belay_interface(*args, **kwargs): res = f(*args, **kwargs) - print(ujson.dumps(res)) + print(json.dumps(res, separators=(',', ':'))) return res globals()["{_BELAY_PREFIX}" + f.__name__] = belay_interface return f @@ -46,6 +46,18 @@ def belay_interface(*args, **kwargs): # Creates and populates two set[str]: all_files, all_dirs _BEGIN_SYNC_CODE = """import os, hashlib, binascii +def __belay_hash_file(fn): + hasher = hashlib.sha256() + try: + with open(fn, "rb") as f: + while True: + data = f.read(4096) + if not data: + break + hasher.update(data) + except OSError: + return "0" * 64 + return str(binascii.hexlify(hasher.digest())) all_files, all_dirs = set(), [] def enumerate_fs(path=""): for elem in os.ilistdir(path): @@ -68,7 +80,7 @@ def enumerate_fs(path=""): os.rmdir(folder) except OSError: pass -del all_files, all_dirs +del all_files, all_dirs, __belay_hash_file """ @@ -145,7 +157,7 @@ def __call__( src_code, src_lineno, src_file = getsource(f) # Add the json_decorator decorator for handling serialization. - src_code = "@json_decorator\n" + src_code + src_code = "@__belay_json\n" + src_code # Send the source code over to the device. self._belay_device(src_code, minify=minify) @@ -332,20 +344,6 @@ def sync( # This is so we know what to clean up after done syncing. self(_BEGIN_SYNC_CODE) - @self.task(register=False) - def remote_hash_file(fn): - hasher = hashlib.sha256() - try: - with open(fn, "rb") as f: # noqa: PL123 - while True: - data = f.read(4096) - if not data: - break - hasher.update(data) - except OSError: - return "0" * 64 - return str(binascii.hexlify(hasher.digest())) - # Sort so that folder creation comes before file sending. local_files = sorted(folder.rglob("*")) for src in local_files: @@ -370,7 +368,7 @@ def remote_hash_file(fn): # All other files, just sync over. local_hash = local_hash_file(src) - remote_hash = remote_hash_file(dst) + remote_hash = self(f"__belay_hash_file({json.dumps(dst)})") if local_hash != remote_hash: self._board.fs_put(src, dst) self(f'all_files.discard("{dst}")') diff --git a/docs/source/How Belay Works.rst b/docs/source/How Belay Works.rst index 468aef1..af4cfaa 100644 --- a/docs/source/How Belay Works.rst +++ b/docs/source/How Belay Works.rst @@ -63,7 +63,7 @@ After minification, the code looks like: The ``0`` is just a one character way of saying ``pass``, in case the removed docstring was the entire body. This reduces the number of transmitted characters from 158 to just 53, offering a 3x speed boost. -After minification, the ``@json_decorator`` is added. On-device, this defines a variant of the function, ``__belay_FUNCTION_NAME`` +After minification, the ``@__belay_json`` decorator is added. On-device, this defines a variant of the function, ``_belay_FUNCTION_NAME`` that performs the following actions: 1. Takes the returned value of the function, and serializes it to json data. Json was chosen since its built into micropython and is "good enough." @@ -79,7 +79,7 @@ Conceptually, its as if the following code ran on-device (minification removed f Pin(25, Pin.OUT).value(state) - def __belay_set_led(*args, **kwargs): + def _belay_set_led(*args, **kwargs): res = set_led(*args, **kwargs) print(json.dumps(res)) @@ -95,7 +95,7 @@ and then parses back the response. The complete lifecycle looks like this: 1. ``set_led(True)`` is called on the host. This doesn't execute the function we defined on host. Instead it triggers the following actions. -2. Belay creates the string ``"__belay_set_led(True)"``. +2. Belay creates the string ``"_belay_set_led(True)"``. 3. Belay sends this command over serial to the REPL, causing it to execute on-device. diff --git a/tests/test_device.py b/tests/test_device.py index 5d196f6..35d1cce 100644 --- a/tests/test_device.py +++ b/tests/test_device.py @@ -1,4 +1,4 @@ -from posixpath import join +import json from unittest.mock import call import pytest @@ -35,18 +35,17 @@ def test_device_task(mocker, mock_device): def foo(a, b): c = a + b # noqa: F841 - mock_device._board.exec.assert_any_call("@json_decorator\ndef foo(a,b):\n c=a+b\n") + mock_device._board.exec.assert_any_call("@__belay_json\ndef foo(a,b):\n c=a+b\n") foo(1, 2) assert ( - mock_device._traceback_execute.call_args.args[-1] - == "__belay_foo(*(1, 2), **{})" + mock_device._traceback_execute.call_args.args[-1] == "_belay_foo(*(1, 2), **{})" ) foo(1, b=2) assert ( mock_device._traceback_execute.call_args.args[-1] - == "__belay_foo(*(1,), **{'b': 2})" + == "_belay_foo(*(1,), **{'b': 2})" ) @@ -137,21 +136,26 @@ def sync_path(tmp_path): def test_device_sync_empty_remote(mocker, mock_device, sync_path): - mock_device._traceback_execute = mocker.MagicMock(return_value="0" * 64) + payload = bytes(json.dumps("0" * 64), encoding="utf8") + mock_device._board.exec = mocker.MagicMock(return_value=payload) mock_device.sync(sync_path) expected_cmds = [ - "__belay_remote_hash_file(*('/alpha.py',), **{})", - "__belay_remote_hash_file(*('/bar.txt',), **{})", - "__belay_remote_hash_file(*('/folder1/file1.txt',), **{})", - "__belay_remote_hash_file(*('/folder1/folder1_1/file1_1.txt',), **{})", - "__belay_remote_hash_file(*('/foo.txt',), **{})", + '__belay_hash_file("/alpha.py")', + '__belay_hash_file("/bar.txt")', + '__belay_hash_file("/folder1/file1.txt")', + '__belay_hash_file("/folder1/folder1_1/file1_1.txt")', + '__belay_hash_file("/foo.txt")', ] - call_args_list = mock_device._traceback_execute.call_args_list - assert len(expected_cmds) == len(call_args_list) - for actual_call, expected_cmd in zip(call_args_list, expected_cmds): - assert actual_call.args[-1] == expected_cmd + call_args_list = mock_device._board.exec.call_args_list[1:] + assert len(expected_cmds) <= len(call_args_list) + for i, expected_cmd in enumerate(expected_cmds): + for actual_call in call_args_list: + if actual_call.args[-1] == expected_cmd: + break + else: + raise Exception(f"cmd {i} not found: {expected_cmd}") mock_device._board.fs_put.assert_has_calls( [ @@ -167,7 +171,7 @@ def test_device_sync_empty_remote(mocker, mock_device, sync_path): def test_device_sync_partial_remote(mocker, mock_device, sync_path): - def __belay_remote_hash_file(fn): + def __belay_hash_file(fn): local_fn = sync_path / fn[1:] if local_fn.stem.endswith("1"): return "0" * 64 @@ -175,7 +179,7 @@ def __belay_remote_hash_file(fn): return belay.device.local_hash_file(local_fn) def side_effect(src_file, src_lineno, name, cmd): - nonlocal __belay_remote_hash_file + nonlocal __belay_hash_file return eval(cmd) mock_device._traceback_execute = mocker.MagicMock(side_effect=side_effect)