From feec5531db76cc906d5afce1c3d8bf9f2fee5886 Mon Sep 17 00:00:00 2001 From: hgromer Date: Fri, 22 Jan 2021 20:56:01 -0500 Subject: [PATCH] refactor how Marshal works --- README.md | 28 ++-- pymarshaler/__init__.py | 6 +- pymarshaler/arg_delegates.py | 140 +++++-------------- pymarshaler/errors.py | 10 +- pymarshaler/marshal.py | 257 +++++++++++++++++++++++------------ setup.py | 2 +- tests/test_marshaling.py | 15 +- 7 files changed, 235 insertions(+), 223 deletions(-) diff --git a/README.md b/README.md index 43f1e42..819305c 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,15 @@ class Test: That's it! We can now marshal, and more importantly, unmarshal this object to and from JSON. ```python -from pymarshaler import marshal +from pymarshaler.marshal import Marshal import json test_instance = Test('foo') -blob = marshal.marshal(test_instance) +blob = Marshal.marshal(test_instance) print(blob) >>> '{name: foo}' +marshal = Marshal() result = marshal.unmarshal(Test, json.loads(blob)) print(result.name) >>> 'foo' @@ -57,14 +58,15 @@ As you can see, adding a nested class is as simple as as adding a basic structur Pymarshaler will fail when encountering an unknown field by default, however you can configure it to ignore unknown fields ```python -from pymarshaler import marshal +from pymarshaler.marshal import Marshal from pymarshaler.arg_delegates import ArgBuilderFactory +marshal = Marshal() blob = {'test': 'foo', 'unused_field': 'blah'} result = marshal.unmarshal(Test, blob) ->>> 'Found unknown field (unused_field: blah). If you would like to skip unknown fields set ArgBuilderFactory.ignore_unknown_fields(True)' +>>> 'Found unknown field (unused_field: blah). If you would like to skip unknown fields create a Marshal object who can skip ignore_unknown_fields' -ArgBuilderFactory.ignore_unknown_fields(True) +marhsal = Marshal(ignore_unknown_fields=True) result = marshal.unmarshal(Test, blob) print(result.name) >>> 'foo' @@ -75,7 +77,7 @@ print(result.name) We can use pymarshaler to handle containers as well. Again we take advantage of python's robust typing system ```python -from pymarshaler import marshal +from pymarshaler.marshal import Marshal from typing import Set import json @@ -84,6 +86,7 @@ class TestContainer: def __int__(self, container: Set[str]): self.container = container +marshal = Marshal() container_instance = TestContainer({'foo', 'bar'}) blob = marshal.marshal(container_instance) print(blob) @@ -99,13 +102,14 @@ Pymarshaler can also handle containers that store user defined types. The `Set[s Pymarshaler also supports default values, and will use any default values supplied in the `__init__` if those values aren't present in the JSON data. ```python -from pymarshaler import marshal +from pymarshaler.marshal import Marshal class TestWithDefault: def __init__(self, name: str = 'foo'): self.name = name +marshal = Marshal() result = marshal.unmarshal(TestWithDefault, {}) print(result.name) >>> 'foo' @@ -115,7 +119,7 @@ Pymarshaler will raise an error if any non-default attributes aren't given Pymarshaler also supports a validate method on creation of the python object. This method will be called before being returned to the user. ```python -from pymarshaler import marshal +from pymarshaler.marshal import Marshal class TestWithValidate: @@ -126,6 +130,7 @@ class TestWithValidate: print(f'My name is {self.name}!') +marshal = Marshal() result = marshal.unmarshal(TestWithValidate, {'name': 'foo'}) >>> 'My name is foo!' ``` @@ -135,8 +140,8 @@ This can be used to validate the python object right at construction, potentiall It's also possible to register your own custom unmarshaler for specific user defined classes. ```python -from pymarshaler.arg_delegates import ArgBuilderDelegate, ArgBuilderFactory -from pymarshaler import marshal +from pymarshaler.arg_delegates import ArgBuilderDelegate +from pymarshaler.marshal import Marshal class ClassWithMessage: @@ -160,7 +165,8 @@ class CustomDelegate(ArgBuilderDelegate): return {'message_obj': ClassWithMessage(data['message'])} -ArgBuilderFactory.register_delegate(ClassWithCustomDelegate, CustomDelegate) +marshal = Marshal() +marshal.register_delegate(ClassWithCustomDelegate, CustomDelegate) result = marshal.unmarshal(ClassWithCustomDelegate, {'message': 'Hello from the custom delegate!'}) print(result.message_obj) >>> 'Hello from the custom delegate!' diff --git a/pymarshaler/__init__.py b/pymarshaler/__init__.py index 139a0bb..7d46ca1 100644 --- a/pymarshaler/__init__.py +++ b/pymarshaler/__init__.py @@ -1,7 +1,7 @@ -__version__ = '0.1.1' -__all__ = ['marshal', 'utils', 'arg_delegates', 'errors'] +__version__ = '0.2.0' +__all__ = ['Marshal', 'utils', 'arg_delegates', 'errors'] -from pymarshaler import marshal +from pymarshaler.marshal import Marshal from pymarshaler import utils from pymarshaler import arg_delegates from pymarshaler import errors diff --git a/pymarshaler/arg_delegates.py b/pymarshaler/arg_delegates.py index 7a2f19d..c0825fe 100644 --- a/pymarshaler/arg_delegates.py +++ b/pymarshaler/arg_delegates.py @@ -4,16 +4,7 @@ import dateutil.parser as parser -from pymarshaler.errors import UnknownFieldError, InvalidDelegateError -from pymarshaler.utils import is_user_defined, is_builtin - - -def _apply_typing(param_type, value: typing.Any) -> typing.Any: - delegate = ArgBuilderFactory.get_delegate(param_type) - result = delegate.resolve(value) - if is_user_defined(param_type): - return param_type(**result) - return result +from pymarshaler.errors import UnknownFieldError class ArgBuilderDelegate: @@ -25,46 +16,56 @@ def resolve(self, data): raise NotImplementedError(f'{ArgBuilderDelegate.__name__} has no implementation of resolve') -class ListArgBuilderDelegate(ArgBuilderDelegate): +class FunctionalArgBuilderDelegate(ArgBuilderDelegate): - def __init__(self, cls): + def __init__(self, cls, func): super().__init__(cls) + self.func = func + + def resolve(self, data): + raise NotImplementedError(f'{FunctionalArgBuilderDelegate.__name__} has no implementation of resolve') + + +class ListArgBuilderDelegate(FunctionalArgBuilderDelegate): + + def __init__(self, cls, func): + super().__init__(cls, func) def resolve(self, data: typing.List): inner_type = self.cls.__args__[0] - return [_apply_typing(inner_type, x) for x in data] + return [self.func(inner_type, x) for x in data] -class SetArgBuilderDelegate(ArgBuilderDelegate): +class SetArgBuilderDelegate(FunctionalArgBuilderDelegate): - def __init__(self, cls): - super().__init__(cls) + def __init__(self, cls, func): + super().__init__(cls, func) def resolve(self, data: typing.Set): inner_type = self.cls.__args__[0] - return {_apply_typing(inner_type, x) for x in data} + return {self.func(inner_type, x) for x in data} -class TupleArgBuilderDelegate(ArgBuilderDelegate): +class TupleArgBuilderDelegate(FunctionalArgBuilderDelegate): - def __init__(self, cls): - super().__init__(cls) + def __init__(self, cls, func): + super().__init__(cls, func) def resolve(self, data: typing.Tuple): inner_type = self.cls.__args__[0] - return (_apply_typing(inner_type, x) for x in data) + return (self.func(inner_type, x) for x in data) -class DictArgBuilderDelegate(ArgBuilderDelegate): +class DictArgBuilderDelegate(FunctionalArgBuilderDelegate): - def __init__(self, cls): - super().__init__(cls) + def __init__(self, cls, func): + super().__init__(cls, func) def resolve(self, data: dict): key_type = self.cls.__args__[0] value_type = self.cls.__args__[1] return { - _apply_typing(key_type, key): _apply_typing(value_type, value) for key, value in data.items() + self.func(key_type, key): self.func(value_type, value) for key, value in data.items() } @@ -89,10 +90,10 @@ def resolve(self, data): return parser.parse(data) -class UserDefinedArgBuilderDelegate(ArgBuilderDelegate): +class UserDefinedArgBuilderDelegate(FunctionalArgBuilderDelegate): - def __init__(self, cls, ignore_unknown_fields: bool, walk_unknown_fields: bool): - super().__init__(cls) + def __init__(self, cls, func, ignore_unknown_fields: bool, walk_unknown_fields: bool): + super().__init__(cls, func) self.ignore_unknown_fields = ignore_unknown_fields self.walk_unknown_fields = walk_unknown_fields @@ -105,11 +106,11 @@ def _resolve(self, cls, data: dict): for key, value in data.items(): if key in unsatisfied: param_type = unsatisfied[key].annotation - args[key] = _apply_typing(param_type, value) + args[key] = self.func(param_type, value) elif not self.ignore_unknown_fields: raise UnknownFieldError(f'Found unknown field ({key}: {value}). ' - 'If you would like to skip unknown fields set ' - 'ArgBuilderFactory.ignore_unknown_fields(True)') + 'If you would like to skip unknown fields ' + 'create a Marshal object who can skip ignore_unknown_fields') elif self.walk_unknown_fields: if isinstance(value, dict): args.update(self._resolve(cls, value)) @@ -118,80 +119,3 @@ def _resolve(self, cls, data: dict): if isinstance(x, dict): args.update(self._resolve(cls, x)) return args - - -class _RegisteredDelegates: - - def __init__(self): - self.registered_delegates = {} - - def register(self, cls, delegate: ArgBuilderDelegate): - self.registered_delegates[cls.__name__] = delegate - - def get(self, cls): - return self.registered_delegates[cls.__name__] - - def contains(self, cls): - try: - return cls.__name__ in self.registered_delegates - except AttributeError: - return False - - -class ArgBuilderFactory: - _walk_unknown_fields = False - - _ignore_unknown_fields = False - - _registered_delegates = _RegisteredDelegates() - - _default_arg_builder_delegates = { - typing.List._name: lambda x: ListArgBuilderDelegate(x), - typing.Set._name: lambda x: SetArgBuilderDelegate(x), - typing.Tuple._name: lambda x: TupleArgBuilderDelegate(x), - typing.Dict._name: lambda x: DictArgBuilderDelegate(x), - "PythonBuiltin": lambda x: BuiltinArgBuilderDelegate(x), - "UserDefined": lambda x: UserDefinedArgBuilderDelegate( - x, - ArgBuilderFactory._ignore_unknown_fields, - ArgBuilderFactory._walk_unknown_fields - ), - "DateTime": lambda: DateTimeArgBuilderDelegate(), - } - - @staticmethod - def walk_unknown_fields(walk: bool): - ArgBuilderFactory._walk_unknown_fields = walk - if walk: - ArgBuilderFactory._ignore_unknown_fields = walk - - @staticmethod - def ignore_unknown_fields(ignore: bool): - ArgBuilderFactory._ignore_unknown_fields = ignore - if not ignore: - ArgBuilderFactory._walk_unknown_fields = ignore - - @staticmethod - def register_delegate(cls, delegate_cls): - ArgBuilderFactory._registered_delegates.register(cls, delegate_cls(cls)) - - @staticmethod - def get_delegate(cls) -> ArgBuilderDelegate: - if ArgBuilderFactory._registered_delegates.contains(cls): - return ArgBuilderFactory._registered_delegates.get(cls) - elif is_user_defined(cls): - return ArgBuilderFactory._default_arg_builder_delegates['UserDefined'](cls) - elif '_name' in cls.__dict__: - return ArgBuilderFactory._safe_get(cls._name)(cls) - elif issubclass(cls, datetime.datetime): - return ArgBuilderFactory._default_arg_builder_delegates['DateTime']() - elif is_builtin(cls): - return ArgBuilderFactory._default_arg_builder_delegates['PythonBuiltin'](cls) - else: - raise InvalidDelegateError(f'No delegate for class {cls}') - - @staticmethod - def _safe_get(name): - if name not in ArgBuilderFactory._default_arg_builder_delegates: - raise InvalidDelegateError(f'Unsupported class type {name}') - return ArgBuilderFactory._default_arg_builder_delegates[name] diff --git a/pymarshaler/errors.py b/pymarshaler/errors.py index 3c4bad0..92bce62 100644 --- a/pymarshaler/errors.py +++ b/pymarshaler/errors.py @@ -1,18 +1,18 @@ -class PymarshallError(RuntimeError): +class PymarshalError(RuntimeError): pass -class UnknownFieldError(PymarshallError): +class UnknownFieldError(PymarshalError): pass -class UnsupportedClassError(PymarshallError): +class UnsupportedClassError(PymarshalError): pass -class InvalidDelegateError(PymarshallError): +class InvalidDelegateError(PymarshalError): pass -class MissingFieldsError(PymarshallError): +class MissingFieldsError(PymarshalError): pass diff --git a/pymarshaler/marshal.py b/pymarshaler/marshal.py index 34eee7b..d7ddd61 100644 --- a/pymarshaler/marshal.py +++ b/pymarshaler/marshal.py @@ -1,96 +1,177 @@ +import datetime import inspect import json +import typing import jsonpickle -from pymarshaler.arg_delegates import ArgBuilderFactory -from pymarshaler.errors import MissingFieldsError - - -def unmarshal_str(cls, data: str): - """ - Reconstruct an instance of type `cls` from a JSON formatted string - :param cls: The class type. Must be a user defined type - :param data: The string JSON data - :return: An instance of the class `cls` - - Example: - - >>> class Test: - - >>> def __init__(self, name: str): - >>> self.name = name - - - >>> data = "{'name': 'foo'}" - >>> test_instance = unmarshal_str(Test,data) - >>> print(test_instance.name) - 'foo' - """ - return unmarshal(cls, json.loads(data)) - - -def unmarshal(cls, data: dict): - """ - Reconstruct an instance of type `cls` from JSON - :param cls: The class type. Must be a user defined type - :param data: The JSON data - :return: An instance of the class `cls` - - Example: - - >>> class Test: - - >>> def __init__(self, name: str): - >>> self.name = name - - - >>> data = {'name': 'foo'} - >>> test_instance = unmarshal(Test, data) - >>> print(test_instance.name) - 'foo' - """ - try: - return _unmarshal(cls, data) - except ValueError: - raise ValueError(f'Failed to pymarshaler {data} to class {cls.__name__}') - - -def marshal(obj, indent=2) -> str: - """ - Convert a class instance to a JSON formatted string - :param obj: The object to convert - :param indent: How to format the JSON. Defaults to an indent of 2 - :return: String JSON representation of the class instance - - Example: - - >>> class Test: - - >>> def __init__(self, name: str): - >>> self.name = name - - - >>> test_instance = Test('foo', indent=0) - >>> data = pymarshaler(test_instance) - >>> print(data) - '{name: foo}' - """ - return jsonpickle.encode(obj, unpicklable=False, indent=indent) - - -def _unmarshal(cls, data: dict): - init_params = inspect.signature(cls.__init__).parameters - args = ArgBuilderFactory.get_delegate(cls).resolve(data) - missing = _get_unsatisfied_args(args, init_params) - if len(missing) > 0: - unfilled = [key for key, param in missing.items() if param.default is inspect.Parameter.empty] - if len(unfilled) > 0: - raise MissingFieldsError(f'Missing required field(s): {", ".join(unfilled)}') - result = cls(**args) - if 'validate' in dir(cls): - result.validate() - return result +from pymarshaler.arg_delegates import ArgBuilderDelegate, ListArgBuilderDelegate, \ + SetArgBuilderDelegate, TupleArgBuilderDelegate, DictArgBuilderDelegate, BuiltinArgBuilderDelegate, \ + UserDefinedArgBuilderDelegate, DateTimeArgBuilderDelegate +from pymarshaler.errors import MissingFieldsError, InvalidDelegateError, PymarshalError +from pymarshaler.utils import is_builtin, is_user_defined + + +class _RegisteredDelegates: + + def __init__(self): + self.registered_delegates = {} + + def register(self, cls, delegate: ArgBuilderDelegate): + self.registered_delegates[cls.__name__] = delegate + + def get(self, cls): + return self.registered_delegates[cls.__name__] + + def contains(self, cls): + try: + return cls.__name__ in self.registered_delegates + except AttributeError: + return False + + +class _ArgBuilderFactory: + + def __init__(self, func, ignore_unknown_fields: bool, walk_unknown_fields: bool): + self._registered_delegates = _RegisteredDelegates() + self._default_arg_builder_delegates = { + typing.List._name: lambda x: ListArgBuilderDelegate(x, func), + typing.Set._name: lambda x: SetArgBuilderDelegate(x, func), + typing.Tuple._name: lambda x: TupleArgBuilderDelegate(x, func), + typing.Dict._name: lambda x: DictArgBuilderDelegate(x, func), + "PythonBuiltin": lambda x: BuiltinArgBuilderDelegate(x), + "UserDefined": lambda x: UserDefinedArgBuilderDelegate( + x, + func, + ignore_unknown_fields, + walk_unknown_fields + ), + "DateTime": lambda: DateTimeArgBuilderDelegate() + } + + def register(self, cls, delegate_cls): + self._registered_delegates.register(cls, delegate_cls(cls)) + + def get_delegate(self, cls) -> ArgBuilderDelegate: + if self._registered_delegates.contains(cls): + return self._registered_delegates.get(cls) + elif is_user_defined(cls): + return self._default_arg_builder_delegates['UserDefined'](cls) + elif '_name' in cls.__dict__: + return self._safe_get(cls._name)(cls) + elif issubclass(cls, datetime.datetime): + return self._default_arg_builder_delegates['DateTime']() + elif is_builtin(cls): + return self._default_arg_builder_delegates['PythonBuiltin'](cls) + else: + raise InvalidDelegateError(f'No delegate for class {cls}') + + def _safe_get(self, name): + if name not in self._default_arg_builder_delegates: + raise InvalidDelegateError(f'Unsupported class type {name}') + return self._default_arg_builder_delegates[name] + + +class Marshal: + + def __init__(self, ignore_unknown_fields: bool = False, walk_unknown_fields: bool = False): + if walk_unknown_fields and ignore_unknown_fields is False: + raise PymarshalError('If walk_unknown_fields is True, ignore_unknown_fields must also be True') + + self._arg_builder_factory = _ArgBuilderFactory( + self._apply_typing, + ignore_unknown_fields, + walk_unknown_fields + ) + + @staticmethod + def marshal(obj, indent=2) -> str: + """ + Convert a class instance to a JSON formatted string + :param obj: The object to convert + :param indent: How to format the JSON. Defaults to an indent of 2 + :return: String JSON representation of the class instance + Example: + >>> class Test: + >>> def __init__(self, name: str): + >>> self.name = name + >>> test_instance = Test('foo', indent=0) + >>> data = Marshal.marshal(test_instance) + >>> print(data) + '{name: foo}' + """ + return jsonpickle.encode(obj, unpicklable=False, indent=indent) + + def unmarshal_str(self, cls, data: str): + """ + Reconstruct an instance of type `cls` from a JSON formatted string + :param cls: The class type. Must be a user defined type + :param data: The string JSON data + :return: An instance of the class `cls` + + Example: + + >>> class Test: + + >>> def __init__(self, name: str): + >>> self.name = name + + >>> marshal = Marshal() + >>> data = "{'name': 'foo'}" + >>> test_instance = marshal.unmarshal_str(Test,data) + >>> print(test_instance.name) + 'foo' + """ + return self.unmarshal(cls, json.loads(data)) + + def unmarshal(self, cls, data: dict): + """ + Reconstruct an instance of type `cls` from a JSON formatted string + :param cls: The class type. Must be a user defined type + :param data: The string JSON data + :return: An instance of the class `cls` + + Example: + + >>> class Test: + + >>> def __init__(self, name: str): + >>> self.name = name + + + >>> marshal = Marshal() + >>> data = "{'name': 'foo'}" + >>> test_instance = marshal.unmarshal_str(Test,data) + >>> print(test_instance.name) + 'foo' + """ + try: + return self._unmarshal(cls, data) + except ValueError: + raise ValueError(f'Failed to pymarshaler {data} to class {cls.__name__}') + + def register_delegate(self, cls, delegate_cls): + self._arg_builder_factory.register(cls, delegate_cls) + + def _unmarshal(self, cls, data: dict): + init_params = inspect.signature(cls.__init__).parameters + args = self._arg_builder_factory.get_delegate(cls).resolve(data) + missing = _get_unsatisfied_args(args, init_params) + if len(missing) > 0: + unfilled = [key for key, param in missing.items() if param.default is inspect.Parameter.empty] + if len(unfilled) > 0: + raise MissingFieldsError(f'Missing required field(s): {", ".join(unfilled)}') + result = cls(**args) + if 'validate' in dir(cls): + result.validate() + return result + + def _apply_typing(self, param_type, value: typing.Any) -> typing.Any: + delegate = self._arg_builder_factory.get_delegate(param_type) + result = delegate.resolve(value) + if is_user_defined(param_type): + return param_type(**result) + return result def _get_unsatisfied_args(current_args: dict, all_params: dict): diff --git a/setup.py b/setup.py index c26938c..4617062 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="pymarshaler", - version="0.1.1", + version="0.2.0", author="Hernan Romer", author_email="nanug33@gmail.com", description="Package to marshal and unmarshal python objects", diff --git a/tests/test_marshaling.py b/tests/test_marshaling.py index 1af54db..fe9db69 100644 --- a/tests/test_marshaling.py +++ b/tests/test_marshaling.py @@ -2,11 +2,12 @@ import unittest from pymarshaler import marshal -from pymarshaler.arg_delegates import ArgBuilderFactory from pymarshaler.errors import MissingFieldsError, UnknownFieldError - +from pymarshaler.marshal import Marshal from tests.test_classes import * +marshal = Marshal() + class TestMarshalling(unittest.TestCase): @@ -14,7 +15,7 @@ def setUp(self) -> None: pass def tearDown(self) -> None: - ArgBuilderFactory.ignore_unknown_fields(False) + marshal = Marshal() def test_simple_marshalling(self): inner = Inner("Inner", 10) @@ -86,7 +87,7 @@ def test_fails_on_unused(self): self.assertRaises(UnknownFieldError, lambda: marshal.unmarshal(Inner, blob)) def test_ignores_unused(self): - ArgBuilderFactory.ignore_unknown_fields(True) + marshal = Marshal(ignore_unknown_fields=True) inner = Inner("Inner", 10) marshalled = marshal.marshal(inner) j = json.loads(marshalled) @@ -103,7 +104,7 @@ def test_validate(self): self.assertRaises(ValidateError, lambda: marshal.unmarshal(ClassWithValidate, {})) def test_custom_delegate(self): - ArgBuilderFactory.register_delegate(ClassWithCustomDelegate, CustomNoneDelegate) + marshal.register_delegate(ClassWithCustomDelegate, CustomNoneDelegate) result = marshal.unmarshal(ClassWithCustomDelegate, {}) self.assertEqual(result, ClassWithCustomDelegate()) @@ -124,7 +125,7 @@ def test_nested_dict_list(self): self.assertEqual(nested, result) def test_walk_unknown(self): - ArgBuilderFactory.walk_unknown_fields(True) + marshal = Marshal(True, True) blob = { 'blah': {'name': 'foo', 'blah2': {'value': 1}} } @@ -133,7 +134,7 @@ def test_walk_unknown(self): def _marshall_and_unmarshall(cls, obj): - marshalled = marshal.marshal(obj) + marshalled = Marshal.marshal(obj) return marshal.unmarshal_str(cls, marshalled)