From e3dcce6f55a15aa04b05c94b1c79709bcc4e5189 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 17 Jan 2023 16:08:51 +0100 Subject: [PATCH 1/4] wip: partial dict import bootstrap --- src/execnet/_dictimport.py | 98 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 src/execnet/_dictimport.py diff --git a/src/execnet/_dictimport.py b/src/execnet/_dictimport.py new file mode 100644 index 00000000..89f16789 --- /dev/null +++ b/src/execnet/_dictimport.py @@ -0,0 +1,98 @@ +# incomplete, to be completed later +# this will replace concat based bootstrap +from __future__ import annotations + +import base64 +import json +import sys +import types +import zlib +from importlib import import_module +from importlib.abc import Loader +from importlib.metadata import Distribution +from importlib.metadata import DistributionFinder +from typing import Any +from typing import IO +from typing import Iterable +from typing import Sequence + + +class DictImporter(DistributionFinder, Loader): + def __init__(self, sources: dict[str, str]): + self.sources = sources + + def find_distributions( + self, context: DistributionFinder.Context = DistributionFinder.Context() + ) -> Iterable[Distribution]: + return [] + + def find_module( + self, fullname: str, path: Sequence[str | bytes] | None = None + ) -> Loader | None: + if fullname in self.sources: + return self + if fullname + ".__init__" in self.sources: + return self + return None + + def load_module(self, fullname): + # print "load_module:", fullname + from types import ModuleType + + try: + s = self.sources[fullname] + is_pkg = False + except KeyError: + s = self.sources[fullname + ".__init__"] + is_pkg = True + + co = compile(s, fullname, "exec") + module = sys.modules.setdefault(fullname, ModuleType(fullname)) + module.__loader__ = self + if is_pkg: + module.__path__ = [fullname] + + exec(co, module.__dict__) + return sys.modules[fullname] + + def get_source(self, name: str) -> str | None: + res = self.sources.get(name) + if res is None: + res = self.sources.get(name + ".__init__") + return res + + +def bootstrap( + modules: dict[str, str], + entry: str, + args: dict[str, Any], + set_argv: list[str] | None, +) -> None: + importer = DictImporter(modules) + sys.meta_path.append(importer) + module, attr = entry.split(":") + loaded_module = import_module(module) + entry_func = getattr(loaded_module, entry) + if set_argv is not None: + sys.argv[1:] = set_argv + entry_func(**args) + + +def bootstrap_stdin(stream: IO) -> None: + bootstrap_args = decode_b85_zip_json(stream.readline()) + bootstrap(**bootstrap_args) + + +def decode_b85_zip_json(encoded: bytes | str): + packed = base64.b85decode(encoded) + unpacked = zlib.decompress(packed) + return json.loads(unpacked) + + +def naive_pack_module(module: types.ModuleType, dist: Distribution): + assert module.__file__ is not None + assert module.__path__ + + +if __name__ == "__main__": + bootstrap_stdin(sys.stdin) From c6da0c4923f80cd355c875830400a1be324b14ab Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 24 Feb 2024 14:46:53 +0100 Subject: [PATCH 2/4] WIP STATE --- src/execnet/_dictimport.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/execnet/_dictimport.py b/src/execnet/_dictimport.py index 89f16789..3175c0c3 100644 --- a/src/execnet/_dictimport.py +++ b/src/execnet/_dictimport.py @@ -4,6 +4,7 @@ import base64 import json +import os import sys import types import zlib @@ -11,20 +12,40 @@ from importlib.abc import Loader from importlib.metadata import Distribution from importlib.metadata import DistributionFinder -from typing import Any from typing import IO +from typing import TYPE_CHECKING +from typing import Any from typing import Iterable from typing import Sequence +if TYPE_CHECKING: + pass + + +class DictDistribution(Distribution): + data: dict[str, str] + + def __init__(self, data: dict[str, str]) -> None: + self.data = data + + def read_text(self, filename): + return self.data[filename] + + def locate_file(self, path: str | os.PathLike[str]) -> os.PathLike[str]: + raise FileNotFoundError(path) + class DictImporter(DistributionFinder, Loader): - def __init__(self, sources: dict[str, str]): + """a limited loader/importer for distributins send via json-lines""" + + def __init__(self, sources: dict[str, str], distribution: DictDistribution): self.sources = sources + self.distribution = distribution def find_distributions( self, context: DistributionFinder.Context = DistributionFinder.Context() ) -> Iterable[Distribution]: - return [] + return [self.distribution] def find_module( self, fullname: str, path: Sequence[str | bytes] | None = None @@ -64,11 +85,12 @@ def get_source(self, name: str) -> str | None: def bootstrap( modules: dict[str, str], + distribution: dict[str, str], entry: str, args: dict[str, Any], set_argv: list[str] | None, ) -> None: - importer = DictImporter(modules) + importer = DictImporter(modules, distribution=DictDistribution(distribution)) sys.meta_path.append(importer) module, attr = entry.split(":") loaded_module = import_module(module) From 6713a550f8e67174d945b9f55c933e31e14e252d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 14 Apr 2024 19:34:49 +0200 Subject: [PATCH 3/4] wip state - type checking and naive packing start - still broken mess --- src/execnet/_dictimport.py | 73 +++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/src/execnet/_dictimport.py b/src/execnet/_dictimport.py index 3175c0c3..86ba5053 100644 --- a/src/execnet/_dictimport.py +++ b/src/execnet/_dictimport.py @@ -5,6 +5,7 @@ import base64 import json import os +import pkgutil import sys import types import zlib @@ -13,13 +14,16 @@ from importlib.metadata import Distribution from importlib.metadata import DistributionFinder from typing import IO -from typing import TYPE_CHECKING from typing import Any from typing import Iterable +from typing import NamedTuple +from typing import NewType +from typing import Protocol from typing import Sequence +from typing import cast +from typing import runtime_checkable -if TYPE_CHECKING: - pass +ModuleName = NewType("ModuleName", str) class DictDistribution(Distribution): @@ -35,56 +39,58 @@ def locate_file(self, path: str | os.PathLike[str]) -> os.PathLike[str]: raise FileNotFoundError(path) +class ModuleInfo(NamedTuple): + name: ModuleName + is_pkg: bool + source: str + + class DictImporter(DistributionFinder, Loader): """a limited loader/importer for distributins send via json-lines""" - def __init__(self, sources: dict[str, str], distribution: DictDistribution): + def __init__( + self, sources: dict[ModuleName, ModuleInfo], distribution: DictDistribution + ): self.sources = sources self.distribution = distribution def find_distributions( - self, context: DistributionFinder.Context = DistributionFinder.Context() + self, context: DistributionFinder.Context | None = None ) -> Iterable[Distribution]: + # TODO: filter return [self.distribution] def find_module( self, fullname: str, path: Sequence[str | bytes] | None = None ) -> Loader | None: - if fullname in self.sources: - return self - if fullname + ".__init__" in self.sources: + if ModuleName(fullname) in self.sources: return self return None - def load_module(self, fullname): + def load_module(self, fullname: str) -> types.ModuleType: # print "load_module:", fullname - from types import ModuleType - try: - s = self.sources[fullname] - is_pkg = False - except KeyError: - s = self.sources[fullname + ".__init__"] - is_pkg = True + info = self.sources[ModuleName(fullname)] - co = compile(s, fullname, "exec") - module = sys.modules.setdefault(fullname, ModuleType(fullname)) + co = compile(info.source, fullname, "exec") + module = sys.modules.setdefault(fullname, types.ModuleType(fullname)) module.__loader__ = self - if is_pkg: + if info.is_pkg: module.__path__ = [fullname] exec(co, module.__dict__) return sys.modules[fullname] def get_source(self, name: str) -> str | None: - res = self.sources.get(name) + res = self.sources.get(ModuleName(name)) if res is None: - res = self.sources.get(name + ".__init__") - return res + return None + else: + return res.source def bootstrap( - modules: dict[str, str], + modules: dict[ModuleName, ModuleInfo], distribution: dict[str, str], entry: str, args: dict[str, Any], @@ -100,7 +106,7 @@ def bootstrap( entry_func(**args) -def bootstrap_stdin(stream: IO) -> None: +def bootstrap_stdin(stream: IO[bytes] | IO[str]) -> None: bootstrap_args = decode_b85_zip_json(stream.readline()) bootstrap(**bootstrap_args) @@ -111,9 +117,26 @@ def decode_b85_zip_json(encoded: bytes | str): return json.loads(unpacked) -def naive_pack_module(module: types.ModuleType, dist: Distribution): +@runtime_checkable +class SourceProvidingLoader(Protocol): + def get_source(self, name: str) -> str: + ... + + +def naive_pack_module(module: types.ModuleType, dist: Distribution) -> object: assert module.__file__ is not None assert module.__path__ + data: dict[ModuleName, ModuleInfo] = {} + for info in pkgutil.walk_packages(module.__path__, f"{module.__name__}."): + spec = info.module_finder.find_spec(info.name, None) + assert spec is not None + loader = cast(SourceProvidingLoader, spec.loader) + + source = loader.get_source(info.name) + data[ModuleName(info.name)] = ModuleInfo( + name=ModuleName(info.name), is_pkg=info.ispkg, source=source + ) + return data if __name__ == "__main__": From 0857e6124c0cf22a0422a0cab8bb35e9be4bfc6e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 14 Apr 2024 20:46:46 +0200 Subject: [PATCH 4/4] ruff --- src/execnet/_dictimport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/execnet/_dictimport.py b/src/execnet/_dictimport.py index 86ba5053..f1e84b1a 100644 --- a/src/execnet/_dictimport.py +++ b/src/execnet/_dictimport.py @@ -119,8 +119,7 @@ def decode_b85_zip_json(encoded: bytes | str): @runtime_checkable class SourceProvidingLoader(Protocol): - def get_source(self, name: str) -> str: - ... + def get_source(self, name: str) -> str: ... def naive_pack_module(module: types.ModuleType, dist: Distribution) -> object: