From ed759dc6e4b9181a2093a7d37f7f84dbfb4edc27 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Thu, 20 Jul 2023 06:49:22 +0000 Subject: [PATCH] Fix clear serializers behaviour after master merge --- parsl/serialize/facade.py | 34 ++++++++++++------- .../test_proxystore_deep_pickle_htex.py | 7 ++-- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/parsl/serialize/facade.py b/parsl/serialize/facade.py index d586cb2960..c51b40b3de 100644 --- a/parsl/serialize/facade.py +++ b/parsl/serialize/facade.py @@ -22,14 +22,21 @@ deserializers = {} -def clear_serializers() -> None: - # does not clear deserializers because remote sending back will have a - # different serializer list (and wants to send back results with one of - # the default 4) so clearing all the deserializers means we cannot receive - # results... - global methods_for_data, methods_for_code - methods_for_code = [] - methods_for_data = [] +# maybe it's weird to want to clear away all serializers rather than just the +# data or just code ones? eg in proxystore test, clearing serializers means +# there is no code serializer... so just data sreializer should be cleared +# (or if only having one kind of serializer, no code/data distinction, then +# this is more consistent, but clearing serializer is still a bit weird in +# that case (maybe for security?) - with inserting one near the start more +# usual? +# def clear_serializers() -> None: +# # does not clear deserializers because remote sending back will have a +# # different serializer list (and wants to send back results with one of +# # the default 4) so clearing all the deserializers means we cannot receive +# # results... +# global methods_for_data, methods_for_code +# methods_for_code = [] +# methods_for_data = [] def unregister_serializer(serializer: SerializerBase) -> None: @@ -121,6 +128,9 @@ def serialize(obj: Any, buffer_threshold: int = int(1e6)) -> bytes: else: methods = methods_for_data + if methods == []: + raise RuntimeError("There are no configured serializers") + for method in methods: try: logger.info(f"BENC: trying serializer {method}") @@ -132,15 +142,13 @@ def serialize(obj: Any, buffer_threshold: int = int(1e6)) -> bytes: else: break - # if no serializer found if result is None: - logger.error("BENC: no serializer returned a result") - raise RuntimeError("BENC: No serializers") + raise RuntimeError("No serializer returned a result") elif isinstance(result, BaseException): - logger.error("BENC: exception from final serializer") + logger.error("Serializer returned an excepton, reraise") raise result else: - logger.info("BENC: serialization complete") + logger.debug("Serialization complete") if len(result) > buffer_threshold: logger.warning(f"Serialized object exceeds buffer threshold of {buffer_threshold} bytes, this could cause overflows") return result diff --git a/parsl/tests/test_serialization/test_proxystore_deep_pickle_htex.py b/parsl/tests/test_serialization/test_proxystore_deep_pickle_htex.py index 324eddd249..5a69ce4430 100644 --- a/parsl/tests/test_serialization/test_proxystore_deep_pickle_htex.py +++ b/parsl/tests/test_serialization/test_proxystore_deep_pickle_htex.py @@ -6,7 +6,7 @@ from parsl.tests.configs.htex_local import fresh_config as local_config from parsl.serialize.base import SerializerBase -from parsl.serialize.facade import serialize, deserialize, register_method_for_data, unregister_serializer, clear_serializers +from parsl.serialize.facade import serialize, deserialize, register_method_for_data, unregister_serializer from parsl.serialize.plugin_proxystore_deep_pickle import create_deep_proxystore_serializer @@ -42,7 +42,10 @@ def __str__(self): def test_proxystore_single_call(): later_c = [v for v in parsl.serialize.facade.methods_for_code] later_d = [v for v in parsl.serialize.facade.methods_for_data] - clear_serializers() + + # clear the data serializers, leaving the code serializers in place + parsl.serialize.facade.methods_for_data = [] + s = create_deep_proxystore_serializer(policy=MyDemo) register_method_for_data(s)