diff --git a/Project.toml b/Project.toml index dbf4ccb..c77f4a0 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TransformSpecifications" uuid = "b92eaff8-8912-4b32-ad24-14d7c692d780" authors = ["Beacon Biosignals, Inc"] -version = "0.1.3" +version = "0.2.0" [deps] Legolas = "741b9549-f6ed-4911-9fbf-4a1c0c97f0cd" diff --git a/docs/base_index.md b/docs/base_index.md index 94f25d1..0b4d792 100644 --- a/docs/base_index.md +++ b/docs/base_index.md @@ -4,7 +4,7 @@ Enabling structured transformations via defined I/O specifications. ## Introduction & Overview -This package provides tools to define explicitly-specified transformation components. Such components can then be used to define pipelines that are themselves composed of individual explicitly-specified components, or facilitate distributed computation. +This package provides tools to define explicitly-specified transformation components. Such components can then be used to define pipelines that are themselves composed of individual explicitly-specified components, or facilitate distributed computation. One primary use-case is in creating explicitly defined pipelines that chain components together. These pipelines are in the form of [directed acyclic graphs (DAGs)](https://en.wikipedia.org/wiki/Directed_acyclic_graph), where each node of the graph is a component, and the edges correspond to data transfers between the components. The graph is "directed" since data flows in one direction (from the outputs of a component to the inputs of another), @@ -46,6 +46,12 @@ Private = false ``` ## `NoThrowTransform` + +[`NoThrowTransform`](@ref)s are a way to wrap a transform such that any errors encountered during the application of the transform will be returned as a [`NoThrowResult`](@ref) rather than thrown as an exception. + +!!! tip "Debugging tip" + To get the stack trace for a violation generated by a [`NoThrowTransform`](@ref), call [`transform_force_throw!`](@ref) on it instead of [`transform!`](@ref). + ```@autodocs Modules = [TransformSpecifications] Pages = ["nothrow.jl"] @@ -53,15 +59,22 @@ Private = false ``` ## `NoThrowDAG` + +[`NoThrowDAG`](@ref)s are a way to compose multiple specified transforms ([`DAGStep`](@ref)) into a DAG, such that any errors errors encountered during the application of the DAG will be returned as a [`NoThrowResult`](@ref) rather than thrown as an exception. + +!!! tip "Debugging tips" + To debug the source of a returned violation from a [`NoThrowDAG`](@ref), call [`transform_force_throw!`](@ref) on it instead of [`transform!`](@ref). Errors (and their stack traces) will be thrown directly, rather than returned nicely as [`NoThrowResult`](@ref)s. Alternatively/additionally, create your DAG from a subset of its constituent steps. Bisecting the full DAG chain can help zero in on errors in DAG construction: e.g., `transform!(NoThrowDAG(steps[1:4]), input)`, etc. + ```@autodocs Modules = [TransformSpecifications] Pages = ["nothrow_dag.jl"] Private = false ``` -Here is the mermaid plot generated for the example DAG in [`NoThrowDAG`](@ref): ## Plotting `NoThrowDAG`s +Here is the mermaid plot generated for the example DAG in [`NoThrowDAG`](@ref): + MERMAID_RAW__TO_BE_REPLACED_VIA_MAKE_JL ```@autodocs diff --git a/src/TransformSpecifications.jl b/src/TransformSpecifications.jl index b9676c1..17588b3 100644 --- a/src/TransformSpecifications.jl +++ b/src/TransformSpecifications.jl @@ -18,7 +18,7 @@ export TransformSpecification include("nothrow.jl") export NoThrowResult, NoThrowTransform, nothrow_succeeded, is_identity_no_throw_transform, - transform_unwrapped!, transform_unwrapped + transform_force_throw!, transform_force_throw include("nothrow_dag.jl") export NoThrowDAG, DAGStep, get_step, input_assembler diff --git a/src/nothrow.jl b/src/nothrow.jl index 0e13689..eca4b3e 100644 --- a/src/nothrow.jl +++ b/src/nothrow.jl @@ -251,7 +251,7 @@ the cause of failure in the output `violations` field. !!! note For debugging purposes, it may be helpful to bypass the "no-throw" feature and - so as to have access to a callstack. To do this, use [`transform_unwrapped!`](@ref) + so as to have access to a callstack. To do this, use [`transform_force_throw!`](@ref) in place of `transform!`. See also: [`convert_spec`](@ref) @@ -293,24 +293,30 @@ function Base.show(io::IO, ntt::NoThrowTransform) end """ - transform_unwrapped!(ntt::NoThrowTransform, input) + transform_force_throw!(ntt::NoThrowTransform, input) Apply [`transform!`](@ref) on inner `ntt.transform_spec`, such that the resultant output will be of type `output_specification(ntt.transform_spec)` rather than a `NoThrowResult`, any failure _will_ result in throwing an error. Utility for debugging `NoThrowTransform`s. -See also: [`transform_unwrapped`](@ref) +See also: [`transform_force_throw`](@ref) """ -transform_unwrapped!(ntt::NoThrowTransform, input) = transform!(ntt.transform_spec, input) +function transform_force_throw!(ntt::NoThrowTransform, input) + return is_identity_no_throw_transform(ntt) ? input : + transform!(ntt.transform_spec, input) +end """ - transform_unwrapped(ntt::NoThrowTransform, input) + transform_force_throw(ntt::NoThrowTransform, input) -Non-mutating implmementation of [`transform_unwrapped!`](@ref); applies +Non-mutating implmementation of [`transform_force_throw!`](@ref); applies `transform(ntt.transform_spec, input)`. """ -transform_unwrapped(ntt::NoThrowTransform, input) = transform(ntt.transform_spec, input) +function transform_force_throw(ntt::NoThrowTransform, input) + return is_identity_no_throw_transform(ntt) ? input : + transform(ntt.transform_spec, input) +end """ identity_no_throw_result(result) -> NoThrowResult diff --git a/src/nothrow_dag.jl b/src/nothrow_dag.jl index ee1370f..46b7a1f 100644 --- a/src/nothrow_dag.jl +++ b/src/nothrow_dag.jl @@ -415,6 +415,8 @@ Return [`NoThrowResult`](@ref) of sequentially [`transform!`](@ref)ing all Before each step, that step's `input_assembler` is called on the results of all previous processing steps; this constructor generates input that conforms to the step's `input_specification`. + +See also: [`transform_force_throw!`](@ref) """ function transform!(dag::NoThrowDAG, input) warnings = String[] @@ -423,7 +425,6 @@ function transform!(dag::NoThrowDAG, input) @debug "Applying step `$name`..." # 1. First, assemble the step's input - InSpec = input_specification(step) input = if i_step == 1 # The initial input record does not need to be constructed---it is # :just: the initial input to the dag at large @@ -439,6 +440,7 @@ function transform!(dag::NoThrowDAG, input) # ...and check that it meets the step's input specification. # (Even though this would happen for "free" inside the step's transform, # we check here first so that we can surface a more informative error message) + InSpec = input_specification(step) try convert_spec(InSpec, input) catch e @@ -461,6 +463,45 @@ function transform!(dag::NoThrowDAG, input) return NoThrowResult(; warnings, result=last(component_results)[2]) end +""" + transform_force_throw!(dag::NoThrowDAG, input) + +Utility for debugging [`NoThrowDAG`](@ref)s by consecutively applying `transform!(step, input)` +on each step, such that the output of each step is of type +`output_specification(step.transform_spec)` rather than a `NoThrowResult`, and any +failure _will_ result in throwing an error. +""" +function transform_force_throw!(dag::NoThrowDAG, input) + component_results = OrderedDict{String,Any}() + for (i_step, (name, step)) in enumerate(dag.step_transforms) + @debug "Applying step `$name`..." + + @debug "...assemble the step's input" + input = if i_step == 1 + # The initial input record does not need to be constructed---it is + # :just: the initial input to the dag at large + input + else + transform!(dag.step_input_assemblers[name], component_results) + end + + @debug "...check that it meets the step's input specification" + # (Even though this would happen for "free" inside the step's transform, + # we check here first so that we can surface a more informative error message) + InSpec = input_specification(step) + try + convert_spec(InSpec, input) + catch + rethrow(ArgumentError("Input to step `$name` doesn't conform to specification `$(InSpec)`")) + end + + @debug "...apply the step's transform" + result = transform_force_throw!(step, input) + component_results[name] = isa(result, NoThrowResult) ? result.result : result + end + return last(component_results)[2] +end + function Base.show(io::IO, c::NoThrowDAG) str = "NoThrowDAG ($(input_specification(c)) => $(result_type(output_specification(c)))):\n" for (i, (k, v)) in enumerate(c.step_transforms) diff --git a/test/nothrow.jl b/test/nothrow.jl index d38ec2f..ba3695b 100644 --- a/test/nothrow.jl +++ b/test/nothrow.jl @@ -164,10 +164,10 @@ end result = transform!(ntt, conforming_input_record) @test nothrow_succeeded(result) - result_unwrapped = transform_unwrapped!(ntt, conforming_input_record) + result_unwrapped = transform_force_throw!(ntt, conforming_input_record) @test isequal(result.result, result_unwrapped) - result_unwrapped2 = transform_unwrapped(ntt, conforming_input_record) + result_unwrapped2 = transform_force_throw(ntt, conforming_input_record) @test isequal(result.result, result_unwrapped) @test isequal(result_unwrapped, result_unwrapped2) end @@ -189,9 +189,9 @@ end @test isequal(result.result, result_nested.result) @test result_nested.warnings == ["woohoo"] - result_unwrapped = transform_unwrapped!(ntt, input_record) + result_unwrapped = transform_force_throw!(ntt, input_record) @test result_unwrapped isa SchemaBV1 - result_nested_unwrapped = transform_unwrapped!(ntt_nested, input_record) + result_nested_unwrapped = transform_force_throw!(ntt_nested, input_record) @test result_nested_unwrapped isa NoThrowResult{SchemaBV1} end @@ -202,7 +202,7 @@ end @test !nothrow_succeeded(result) @test startswith(only(result.violations), "Input doesn't conform to specification `SchemaAV1`. Details: ") - @test_throws ArgumentError transform_unwrapped!(ntt, nonconforming_input_record) + @test_throws ArgumentError transform_force_throw!(ntt, nonconforming_input_record) end @testset "Nonconforming transform fails" begin @@ -214,7 +214,8 @@ end @test !nothrow_succeeded(result) @test isequal(only(result.violations), "Unexpected violation. Details: $err") - @test_throws ErrorException transform_unwrapped!(ntt_unexpected_throw, input_record) + @test_throws ErrorException transform_force_throw!(ntt_unexpected_throw, + input_record) end @testset "Nonconforming output fails" begin @@ -225,7 +226,7 @@ end @test !nothrow_succeeded(result) @test isequal(only(result.violations), "Output doesn't conform to specification `NoThrowResult{SchemaAV1}`; is instead a `NoThrowResult{SchemaBV1}`") - @test_throws ErrorException transform_unwrapped!(ntt_expected_throw, input_record) + @test_throws ErrorException transform_force_throw!(ntt_expected_throw, input_record) ntt_nonconforming_out = NoThrowTransform(SchemaAV1, SchemaBV1, input -> NoThrowResult(input)) @@ -233,8 +234,8 @@ end @test !nothrow_succeeded(result) @test isequal(only(result.violations), "Output doesn't conform to specification `NoThrowResult{SchemaBV1}`; is instead a `NoThrowResult{SchemaAV1}`") - @test_throws ErrorException transform_unwrapped!(ntt_nonconforming_out, - input_record) + @test_throws ErrorException transform_force_throw!(ntt_nonconforming_out, + input_record) end @testset "Warnings forwarded" begin @@ -246,7 +247,7 @@ end @test result isa NoThrowResult{SchemaBV1} @test result.warnings == ["Okay now..."] - result_unwrapped = transform_unwrapped!(ntt_warn, SchemaAV1(; foo="rabbit")) + result_unwrapped = transform_force_throw!(ntt_warn, SchemaAV1(; foo="rabbit")) # For non-NoThrowResult output specs, we'd expect # isequal(result.result, result_unwrapped)--- diff --git a/test/nothrow_dag.jl b/test/nothrow_dag.jl index 0dafe9a..d8ec019 100644 --- a/test/nothrow_dag.jl +++ b/test/nothrow_dag.jl @@ -42,8 +42,7 @@ end ntt = NoThrowTransform(SchemaBarV1) @test NoThrowDAG([DAGStep("a", nothing, ntt)]) isa NoThrowDAG err = ArgumentError("Initial step's input constructor must be `nothing` (TransformSpecification{Dict{String, Any},NamedTuple}: `identity`)") - @test_throws err NoThrowDAG([DAGStep("a", input_assembler(identity), - ntt)]) + @test_throws err NoThrowDAG([DAGStep("a", input_assembler(identity), ntt)]) end @testset "Invalid input assembler" begin @@ -126,6 +125,9 @@ end @test !(conforming_input_record isa input_specification(dag)) dag_output2 = transform!(dag, conforming_input_record) @test isequal(dag_output, dag_output2) + + unwrapped_output = transform_force_throw!(dag, conforming_input_record) + @test isequal(dag_output.result, unwrapped_output) end @testset "Nonconforming input fails" begin @@ -135,6 +137,9 @@ end Details: ArgumentError(\"Invalid value set for field `foo`, expected String, \ got a value of type Missing (missing)\")" @test isequal(err_str, only(result.violations)) + + err = ArgumentError("Input to step `step_a` doesn't conform to specification `SchemaFooV1`") + @test_throws err transform_force_throw!(dag, SchemaBarV1(; var1="yay", var2="whee")) end @testset "`_validate_input_assembler`" begin