diff --git a/.gitignore b/.gitignore index db4561ea..ce3d241b 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,8 @@ docs/_build/ # PyBuilder target/ + +# Plots genersated when running sample code +/*.png +/*.svg +/*.pdf \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index d8657a8f..eccdd493 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,19 +2,35 @@ language: python python: - "2.7" - - "3.4" - "3.5" + - "3.6" + - "3.7" + +addons: + apt: + packages: + - graphviz + install: - - pip install Sphinx sphinx_rtd_theme codecov packaging + - pip install Sphinx codecov packaging - "python -c $'import os, packaging.version as version\\nv = version.parse(os.environ.get(\"TRAVIS_TAG\", \"1.0\")).public\\nwith open(\"VERSION\", \"w\") as f: f.write(v)'" - - python setup.py install + - pip install -e .[test] - cd docs - make clean html - cd .. script: - - python setup.py nosetests --with-coverage --cover-package=graphkit + # OVERRIDE pytest-defaults adopted in `setup.cfg`: + # + # Run doctests in latest Python; certainly not < PY3.6 due to unstable dicts. + # Also give `-m 'slow or not slow'` since `not slow` adopted in `setup.cfg`. + - | + if [[ "$TRAVIS_PYTHON_VERSION" = '3.7' ]]; then + pytest --cov=graphkit -m 'slow or not slow' + else + pytest --cov=graphkit test/ + fi deploy: provider: pypi diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 00000000..381f3338 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,208 @@ +######### +Changelog +######### + +v1.3.0 (Oct 2019): New DAG solver, better plotting & "sideffect" +================================================================ + +Kept external API (hopefully) the same, but revamped pruning algorithm and +refactored network compute/compile structure, so results may change; significantly +enhanced plotting. The only new feature actually is the :class:`sideffect`` modifier. + +Network: +-------- + ++ FIX(:gh:`18`, :gh:`26`, :gh:`29`, :gh:`17`, :gh:`20`): Revamped DAG SOLVER + to fix bad pruning described in :gh:`24` & :gh:`25` + + Pruning now works by breaking incoming provide-links to any given + intermedediate inputs dropping operations with partial inputs or without outputs. + + The end result is that operations in the graph that do not have all inputs satisfied, + they are skipped (in v1.2.4 they crashed). + + Also started annotating edges with optional/sideffects, to make proper use of + the underlying ``networkx`` graph. + + |v130-flowchart| + ++ REFACT(:gh:`21`, :gh:`29`): Refactored Network and introduced :class:`ExecutionPlan` to keep + compilation results (the old ``steps`` list, plus input/output names). + + Moved also the check for when to evict a value, from running the execution-plan, + to whenbuilding it; thus, execute methods don't need outputs anymore. + ++ ENH(:gh:`26`): "Pin* input values that may be overriten by calculated ones. + + This required the introduction of the new :class:`PinInstruction` in + the execution plan. + ++ FIX(:gh:`23`, :gh:`22`-2.4.3): Keep consistent order of ``networkx.DiGraph`` + and *sets*, to generate deterministic solutions. + + *Unfortunately*, it non-determinism has not been fixed in < PY3.5, just + reduced the frequency of `spurious failures + `_, caused by + unstable dicts, and the use of subgraphs. + ++ enh: Mark outputs produced by :class:`NetworkOperation`'s needs as ``optional``. + TODO: subgraph network-operations would not be fully functional until + *"optional outpus"* are dealt with (see :gh:`22`-2.5). + ++ enh: Annotate operation exceptions with ``ExecutionPlan`` to aid debug sessions, + ++ drop: methods ``list_layers()``/``show layers()`` not needed, ``repr()`` is + a better replacement. + + +Plotting: +--------- + ++ ENH(:gh:`13`, :gh:`26`, :gh:`29`): Now network remembers last plan and uses that + to overlay graphs with the internals of the planing and execution: |sample-plot| + + + - execution-steps & order + - delete & pin instructions + - given inputs & asked outputs + - solution values (just if they are present) + - "optional" needs & broken links during pruning + ++ REFACT: Move all API doc on plotting in a single module, splitted in 2 phases, + build DOT & render DOT + ++ FIX(:gh:`13`): bring plot writing into files up-to-date from PY2; do not create plot-file + if given file-extension is not supported. + ++ FEAT: path `pydot library `_ to support rendering + in *Jupyter notebooks*. + + + +Testing & other code: +--------------------- + + - Increased coverage from 77% --> 90%. + ++ ENH(:gh:`28`): use ``pytest``, to facilitate TCs parametrization. + ++ ENH(:gh:`30`): Doctest all code; enabled many assertions that were just print-outs + in v1.2.4. + ++ FIX: ``operation.__repr__()`` was crashing when not all arguments + had been set - a condition frequtnly met during debugging session or failed + TCs (inspired by @syamajala's 309338340). + ++ enh: Sped up parallel/multihtread TCs by reducing delays & repetitions. + + .. tip:: + You need ``pytest -m slow`` to run those slow tests. + + + +Chore & Docs: +------------- + ++ FEAT: add changelog in ``CHANGES.rst`` file, containing flowcharts + to compare versions ``v1.2.4 <--> v1.3..0``. ++ enh: updated site & documentation for all new features, comparing with v1.2.4. ++ enh(:gh:`30`): added "API reference' chapter. ++ drop(build): ``sphinx_rtd_theme`` library is the default theme for Sphinx now. ++ enh(build): Add ``test`` *pip extras*. ++ sound: https://www.youtube.com/watch?v=-527VazA4IQ, + https://www.youtube.com/watch?v=8J182LRi8sU&t=43s + + + +v1.2.4 (Mar 7, 2018) +==================== + ++ Issues in pruning algorithm: :gh:`24`, :gh:`25` ++ Blocking bug in plotting code for Python-3.x. ++ Test-cases without assertions (just prints). + +|v124-flowchart| + + + +1.2.2 (Mar 7, 2018, @huyng): Fixed versioning +============================================= + +Versioning now is manually specified to avoid bug where the version +was not being correctly reflected on pip install deployments + + + +1.2.1 (Feb 23, 2018, @huyng): Fixed multi-threading bug and faster compute through caching of `find_necessary_steps` +==================================================================================================================== + +We've introduced a cache to avoid computing find_necessary_steps multiple times +during each inference call. + +This has 2 benefits: + ++ It reduces computation time of the compute call ++ It avoids a subtle multi-threading bug in networkx when accessing the graph + from a high number of threads. + + + +1.2.0 (Feb 13, 2018, @huyng) +============================ + +Added `set_execution_method('parallel')` for execution of graphs in parallel. + + +1.1.0 (Nov 9, 2017, @huyng) +=========================== + +Update setup.py + + +1.0.4 (Nov 3, 2017, @huyng): Networkx 2.0 compatibility +======================================================= + +Minor Bug Fixes: + ++ Compatibility fix for networkx 2.0 ++ `net.times` now only stores timing info from the most recent run + + +1.0.3 (Jan 31, 2017, @huyng): Make plotting dependencies optional +================================================================= + ++ Merge pull request :gh:`6` from yahoo/plot-optional ++ make plotting dependencies optional + + +1.0.2 (Sep 29, 2016, @pumpikano): Merge pull request :gh:`5` from yahoo/remove-packaging-dep +============================================================================================ + ++ Remove 'packaging' as dependency + + +1.0.1 (Aug 24, 2016) +==================== + +1.0 (Aug 2, 2016, @robwhess) +============================ + +First public release in PyPi & GitHub. + ++ Merge pull request :gh:`3` from robwhess/travis-build ++ Travis build + + +.. _substitutions: + + +.. |sample-plot| image:: docs/source/images/sample_plot.svg + :alt: sample graphkit plot + :width: 120px + :align: bottom +.. |v130-flowchart| image:: docs/source/images/GraphkitFlowchart-v1.3.0.svg + :alt: graphkit-v1.3.0 flowchart + :scale: 75% +.. |v124-flowchart| image:: docs/source/images/GraphkitFlowchart-v1.2.4.svg + :alt: graphkit-v1.2.4 flowchart + :scale: 75% diff --git a/README.md b/README.md index 0e1e95a4..a04d58db 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,91 @@ # GraphKit -[![PyPI version](https://badge.fury.io/py/graphkit.svg)](https://badge.fury.io/py/graphkit) [![Build Status](https://travis-ci.org/yahoo/graphkit.svg?branch=master)](https://travis-ci.org/yahoo/graphkit) [![codecov](https://codecov.io/gh/yahoo/graphkit/branch/master/graph/badge.svg)](https://codecov.io/gh/yahoo/graphkit) +[![Latest version in PyPI](https://img.shields.io/pypi/v/graphkit.svg?label=PyPi%20version)](https://img.shields.io/pypi/v/graphkit.svg?label=PyPi%20version) +[![Latest version in GitHub](https://img.shields.io/github/v/release/yahoo/graphkit.svg?label=GitHub%20release&include_prereleases)](https://img.shields.io/github/v/release/yahoo/graphkit.svg?label=GitHub%20release&include_prereleases) +[![Supported Python versions of latest release in PyPi](https://img.shields.io/pypi/pyversions/graphkit.svg?label=Python)](https://img.shields.io/pypi/pyversions/graphkit.svg?label=Python) +[![Build Status](https://travis-ci.org/yahoo/graphkit.svg?branch=master)](https://travis-ci.org/yahoo/graphkit) +[![codecov](https://codecov.io/gh/yahoo/graphkit/branch/master/graph/badge.svg)](https://codecov.io/gh/yahoo/graphkit) +[![License](https://img.shields.io/pypi/l/graphkit.svg)](https://img.shields.io/pypi/l/graphkit.svg) + +[![Github watchers](https://img.shields.io/github/watchers/yahoo/graphkit.svg?style=social)](https://img.shields.io/github/watchers/yahoo/graphkit.svg?style=social) +[![Github stargazers](https://img.shields.io/github/stars/yahoo/graphkit.svg?style=social)](https://img.shields.io/github/stars/yahoo/graphkit.svg?style=social) +[![Github forks](https://img.shields.io/github/forks/yahoo/graphkit.svg?style=social)](https://img.shields.io/github/forks/yahoo/graphkit.svg?style=social) +[![Issues count](http://img.shields.io/github/issues/yahoo/graphkit.svg?style=social)](http://img.shields.io/github/issues/yahoo/graphkit.svg?style=social) [Full Documentation](https://pythonhosted.org/graphkit/) > It's a DAG all the way down +![Sample graph](docs/source/images/test_pruning_not_overrides_given_intermediate-asked.png "Sample graph") + ## Lightweight computation graphs for Python -GraphKit is a lightweight Python module for creating and running ordered graphs of computations, where the nodes of the graph correspond to computational operations, and the edges correspond to output --> input dependencies between those operations. Such graphs are useful in computer vision, machine learning, and many other domains. +GraphKit is an an understandable and lightweight Python module for building and running +ordered graphs of computations. +The API posits a fair compromise between features and complexity without precluding any. +It might be of use in computer vision, machine learning and other data science domains, +or become the core of a custom ETL pipelne. ## Quick start Here's how to install: -``` -pip install graphkit -``` + pip install graphkit -Here's a Python script with an example GraphKit computation graph that produces multiple outputs (`a * b`, `a - a * b`, and `abs(a - a * b) ** 3`): +OR with dependencies for plotting support (and you need to install [`Graphviz`](https://graphviz.org) +program separately with your OS tools): -``` -from operator import mul, sub -from graphkit import compose, operation + pip install graphkit[plot] -# Computes |a|^p. -def abspow(a, p): - c = abs(a) ** p - return c +Here's a Python script with an example GraphKit computation graph that produces +multiple outputs (`a * b`, `a - a * b`, and `abs(a - a * b) ** 3`): -# Compose the mul, sub, and abspow operations into a computation graph. -graph = compose(name="graph")( - operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), - operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), - operation(name="abspow1", needs=["a_minus_ab"], provides=["abs_a_minus_ab_cubed"], params={"p": 3})(abspow) -) + >>> from operator import mul, sub + >>> from graphkit import compose, operation -# Run the graph and request all of the outputs. -out = graph({'a': 2, 'b': 5}) + >>> # Computes |a|^p. + >>> def abspow(a, p): + ... c = abs(a) ** p + ... return c -# Prints "{'a': 2, 'a_minus_ab': -8, 'b': 5, 'ab': 10, 'abs_a_minus_ab_cubed': 512}". -print(out) + >>> # Compose the mul, sub, and abspow operations into a computation graph. + >>> graphop = compose(name="graphop")( + ... operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), + ... operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), + ... operation(name="abspow1", needs=["a_minus_ab"], provides=["abs_a_minus_ab_cubed"], params={"p": 3})(abspow) + ... ) -# Run the graph and request a subset of the outputs. -out = graph({'a': 2, 'b': 5}, outputs=["a_minus_ab"]) + >>> # Run the graph and request all of the outputs. + >>> out = graphop({'a': 2, 'b': 5}) + >>> print(out) + {'a': 2, 'b': 5, 'ab': 10, 'a_minus_ab': -8, 'abs_a_minus_ab_cubed': 512} -# Prints "{'a_minus_ab': -8}". -print(out) -``` + >>> # Run the graph and request a subset of the outputs. + >>> out = graphop({'a': 2, 'b': 5}, outputs=["a_minus_ab"]) + >>> print(out) + {'a_minus_ab': -8} + As you can see, any function can be used as an operation in GraphKit, even ones imported from system modules! + +## Plotting + +For debugging the above graph-operation you may plot the *execution plan* +of the last computation it using these methods: + +```python +graphop.plot(show=True) # open a matplotlib window +graphop.plot("intro.svg") # other supported formats: png, jpg, pdf, ... +graphop.plot() # without arguments return a pydot.DOT object +graphop.plot(solution=out) # annotate graph with solution values +``` + +![Intro graph](docs/source/images/intro.svg "Intro graph") +![Graphkit Legend](docs/source/images/GraphkitLegend.svg "Graphkit Legend") + +> **TIP:** The `pydot.Dot` instances returned by `plot()` are rendered as SVG in *Jupyter/IPython*. + # License Code licensed under the Apache License, Version 2.0 license. See LICENSE file for terms. diff --git a/bin/build.sh b/bin/build.sh new file mode 100755 index 00000000..d3aea133 --- /dev/null +++ b/bin/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# clean, or invalid files in packages +rm -vrf ./build/* ./dist/* ./*.pyc ./*.tgz ./*.egg-info +python setup.py sdist bdist_wheel + diff --git a/docs/GraphkitFlowchart.dot b/docs/GraphkitFlowchart.dot new file mode 100644 index 00000000..af1217f3 --- /dev/null +++ b/docs/GraphkitFlowchart.dot @@ -0,0 +1,31 @@ +# Render it manually with this command, and remember to update result in git: +# +# dot docs/GraphkitFlowchart.dot -Tsvg -odocs/source/images/GraphkitFlowchart-vX.Y.Z.svg +# +digraph { + label="graphkit-v1.3.0 flowchart"; + labelloc=t; + + operations [shape=parallelogram fontname="italic"]; + compose [fontname="italic"]; + network [shape=parallelogram fontname="italic"]; + inputs [shape=rect label="input names"]; + outputs [shape=rect label="output names"]; + subgraph cluster_compute { + label=compute + fontname=bold + style=dashed; + labelloc=b; + + compile [fontname="italic"]; + plan [shape=parallelogram label="execution plan" fontname="italic"]; + execute [fontname=italic fontname="italic"]; + } + values [shape=rect label="input values"]; + solution [shape=rect]; + overwrites [shape=rect]; + + operations -> compose -> network [arrowhead=vee]; + {network inputs outputs} -> compile -> plan [arrowhead=vee]; + {plan values} -> execute -> {solution overwrites} [arrowhead=vee]; +} \ No newline at end of file diff --git a/docs/source/changes.rst b/docs/source/changes.rst new file mode 100644 index 00000000..63553b51 --- /dev/null +++ b/docs/source/changes.rst @@ -0,0 +1,13 @@ +.. include:: ../../CHANGES.rst + :end-before: .. _substitutions: + +.. |sample-plot| image:: images/sample_plot.svg + :alt: sample graphkit plot + :width: 120px + :align: bottom +.. |v130-flowchart| image:: images/GraphkitFlowchart-v1.3.0.svg + :alt: graphkit-v1.3.0 flowchart + :scale: 75% +.. |v124-flowchart| image:: images/GraphkitFlowchart-v1.2.4.svg + :alt: graphkit-v1.2.4 flowchart + :scale: 75% diff --git a/docs/source/composition.rst b/docs/source/composition.rst new file mode 100644 index 00000000..c86bf0c0 --- /dev/null +++ b/docs/source/composition.rst @@ -0,0 +1,206 @@ +.. _graph-composition: + +Graph Composition +================= + +GraphKit's ``compose`` class handles the work of tying together ``operation`` instances into a runnable computation graph. + +The ``compose`` class +--------------------- + +For now, here's the specification of ``compose``. We'll get into how to use it in a second. + +.. autoclass:: graphkit.compose + :members: __call__ + :special-members: + + +.. _simple-graph-composition: + +Simple composition of operations +-------------------------------- + +The simplest use case for ``compose`` is assembling a collection of individual operations into a runnable computation graph. The example script from :ref:`quick-start` illustrates this well:: + + >>> from operator import mul, sub + >>> from graphkit import compose, operation + + >>> # Computes |a|^p. + >>> def abspow(a, p): + ... c = abs(a) ** p + ... return c + + >>> # Compose the mul, sub, and abspow operations into a computation graph. + >>> graphop = compose(name="graphop")( + ... operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), + ... operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), + ... operation(name="abspow1", needs=["a_minus_ab"], provides=["abs_a_minus_ab_cubed"], params={"p": 3})(abspow) + ... ) + +The call here to ``compose()`` yields a runnable computation graph that looks like this (where the circles are operations, squares are data, and octagons are parameters): + +.. image:: images/intro.svg + + +.. _graph-computations: + +Running a computation graph +--------------------------- + +The graph composed in the example above in :ref:`simple-graph-composition` can be run +by simply calling it with a dictionary argument whose keys correspond to the names of inputs +to the graph and whose values are the corresponding input values. +For example, if ``graph`` is as defined above, we can run it like this:: + + # Run the graph and request all of the outputs. + >>> out = graphop({'a': 2, 'b': 5}) + >>> out + {'a': 2, 'b': 5, 'ab': 10, 'a_minus_ab': -8, 'abs_a_minus_ab_cubed': 512} + +Producing a subset of outputs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, calling a graph-operation on a set of inputs will yield all of that graph's outputs. You can use the ``outputs`` parameter to request only a subset. For example, if ``graphop`` is as above:: + + # Run the graph-operation and request a subset of the outputs. + >>> out = graphop({'a': 2, 'b': 5}, outputs=["a_minus_ab"]) + >>> out + {'a_minus_ab': -8} + +When using ``outputs`` to request only a subset of a graph's outputs, GraphKit executes only the ``operation`` nodes in the graph that are on a path from the inputs to the requested outputs. For example, the ``abspow1`` operation will not be executed here. + +Short-circuiting a graph computation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can short-circuit a graph computation, making certain inputs unnecessary, by providing a value in the graph that is further downstream in the graph than those inputs. For example, in the graph-operation we've been working with, you could provide the value of ``a_minus_ab`` to make the inputs ``a`` and ``b`` unnecessary:: + + # Run the graph-operation and request a subset of the outputs. + >>> out = graphop({'a_minus_ab': -8}) + >>> out + {'a_minus_ab': -8, 'abs_a_minus_ab_cubed': 512} + +When you do this, any ``operation`` nodes that are not on a path from the downstream input to the requested outputs (i.e. predecessors of the downstream input) are not computed. For example, the ``mul1`` and ``sub1`` operations are not executed here. + +This can be useful if you have a graph-operation that accepts alternative forms of the same input. For example, if your graph-operation requires a ``PIL.Image`` as input, you could allow your graph to be run in an API server by adding an earlier ``operation`` that accepts as input a string of raw image data and converts that data into the needed ``PIL.Image``. Then, you can either provide the raw image data string as input, or you can provide the ``PIL.Image`` if you have it and skip providing the image data string. + +Adding on to an existing computation graph +------------------------------------------ + +Sometimes you will have an existing computation graph to which you want to add operations. This is simple, since ``compose`` can compose whole graphs along with individual ``operation`` instances. For example, if we have ``graph`` as above, we can add another operation to it to create a new graph:: + + >>> # Add another subtraction operation to the graph. + >>> bigger_graph = compose(name="bigger_graph")( + ... graphop, + ... operation(name="sub2", needs=["a_minus_ab", "c"], provides="a_minus_ab_minus_c")(sub) + ... ) + + >>> # Run the graph and print the output. + >>> sol = bigger_graph({'a': 2, 'b': 5, 'c': 5}, outputs=["a_minus_ab_minus_c"]) + >>> sol + {'a_minus_ab_minus_c': -13} + +This yields a graph which looks like this (see :ref:`plotting`): + +.. image:: images/bigger_example_graph.svg + + + +More complicated composition: merging computation graphs +---------------------------------------------------------- + +Sometimes you will have two computation graphs---perhaps ones that share operations---you want to combine into one. In the simple case, where the graphs don't share operations or where you don't care whether a duplicated operation is run multiple (redundant) times, you can just do something like this:: + + combined_graph = compose(name="combined_graph")(graph1, graph2) + +However, if you want to combine graphs that share operations and don't want to pay the price of running redundant computations, you can set the ``merge`` parameter of ``compose()`` to ``True``. This will consolidate redundant ``operation`` nodes (based on ``name``) into a single node. For example, let's say we have ``graphop``, as in the examples above, along with this graph:: + + >>> # This graph shares the "mul1" operation with graph. + >>> another_graph = compose(name="another_graph")( + ... operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), + ... operation(name="mul2", needs=["c", "ab"], provides=["cab"])(mul) + ... ) + +We can merge ``graphop`` and ``another_graph`` like so, avoiding a redundant ``mul1`` operation:: + + >>> merged_graph = compose(name="merged_graph", merge=True)(graphop, another_graph) + >>> print(merged_graph) + NetworkOperation(name='merged_graph', + needs=[optional('a'), optional('b'), optional('c')], + provides=['ab', 'a_minus_ab', 'abs_a_minus_ab_cubed', 'cab']) + +This ``merged_graph`` will look like this: + +.. image:: images/example_merged_graph.svg + +As always, we can run computations with this graph by simply calling it:: + + >>> merged_graph({'a': 2, 'b': 5, 'c': 5}, outputs=["cab"]) + {'cab': 50} + + + +Errors & debugging +------------------ + +If an operation fails, the original exception gets annotated +with the folllowing properties, as a debug aid: + +>>> from pprint import pprint + +>>> def scream(*args): +... raise ValueError("Wrong!") + +>>> try: +... compose("errgraph")( +... operation(name="screamer", needs=['a'], provides=["foo"])(scream) +... )({'a': None}) +... except ValueError as ex: +... pprint(ex.graphkit_aid) +{'network': + ... + 'operation': FunctionalOperation(name='screamer', needs=['a'], provides=['foo']), + 'operation_args': {'args': [None], 'kwargs': {}}, + 'operation_fnouts': None, + 'operation_outs': None, + 'operation_results': None, + 'plan': ExecutionPlan(inputs=('a',), outputs=(), steps: + +--FunctionalOperation(name='screamer', needs=['a'], provides=['foo'])), + 'solution': {'a': None}} + + +The following annotated attributes might have values on an exception ``ex``: + +``ex.network`` + the innermost network owning the failed operation/function + +``ex.plan`` + the innermost plan that executing when a operation crashed + +``ex.operation`` + the innermost operation that failed + +``ex.operation_args`` + either a 2-tuple ``(args, kwargs)`` or just the ``args`` fed to the operation + +``ex.operation_fnouts`` + the names of the outputs the function was expected to return + +``ex.operation_outs`` + the names eventually the graph needed from the operation + (a subset of the above) + +``ex.operation_results`` + the values dict, if any; it maybe a *zip* of the provides + with the actual returned values of the function, ot the raw results. + +.. note:: + The :ref:`plotting` capabilities, along with the above annotation of exceptions + with the internal state of plan/operation often renders a debugger session + unnecessary. But since the state of the annotated values might be incomple, + you may not always avoid one. + + +Execution internals +------------------- +.. automodule:: graphkit.network + :noindex: diff --git a/docs/source/conf.py b/docs/source/conf.py index a92cff6f..859419e9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,6 @@ import sys import os -import sphinx_rtd_theme import packaging.version # If extensions (or modules to document with autodoc) are in another directory, @@ -36,9 +35,14 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.coverage', - 'sphinx.ext.imgmath' + 'sphinx.ext.imgmath', + 'sphinx.ext.extlinks', ] +extlinks = { + 'gh': ('https://github.com/yahoo/graphkit/issues/%s', '#'), +} + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -109,11 +113,6 @@ # -- Options for HTML output ---------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. diff --git a/docs/source/graph_composition.rst b/docs/source/graph_composition.rst deleted file mode 100644 index ba428a14..00000000 --- a/docs/source/graph_composition.rst +++ /dev/null @@ -1,131 +0,0 @@ -.. _graph-composition: - -Graph Composition and Use -========================= - -GraphKit's ``compose`` class handles the work of tying together ``operation`` instances into a runnable computation graph. - -The ``compose`` class ---------------------- - -For now, here's the specification of ``compose``. We'll get into how to use it in a second. - -.. autoclass:: graphkit.compose - :members: __call__ - - -.. _simple-graph-composition: - -Simple composition of operations --------------------------------- - -The simplest use case for ``compose`` is assembling a collection of individual operations into a runnable computation graph. The example script from :ref:`quick-start` illustrates this well:: - - from operator import mul, sub - from graphkit import compose, operation - - # Computes |a|^p. - def abspow(a, p): - c = abs(a) ** p - return c - - # Compose the mul, sub, and abspow operations into a computation graph. - graph = compose(name="graph")( - operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), - operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), - operation(name="abspow1", needs=["a_minus_ab"], provides=["abs_a_minus_ab_cubed"], params={"p": 3})(abspow) - ) - -The call here to ``compose()()`` yields a runnable computation graph that looks like this (where the circles are operations, squares are data, and octagons are parameters): - -.. image:: images/example_graph.svg - - -.. _graph-computations: - -Running a computation graph ---------------------------- - -The graph composed in the example above in :ref:`simple-graph-composition` can be run by simply calling it with a dictionary argument whose keys correspond to the names of inputs to the graph and whose values are the corresponding input values. For example, if ``graph`` is as defined above, we can run it like this:: - - # Run the graph and request all of the outputs. - out = graph({'a': 2, 'b': 5}) - - # Prints "{'a': 2, 'a_minus_ab': -8, 'b': 5, 'ab': 10, 'abs_a_minus_ab_cubed': 512}". - print(out) - -Producing a subset of outputs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, calling a graph on a set of inputs will yield all of that graph's outputs. You can use the ``outputs`` parameter to request only a subset. For example, if ``graph`` is as above:: - - # Run the graph and request a subset of the outputs. - out = graph({'a': 2, 'b': 5}, outputs=["a_minus_ab"]) - - # Prints "{'a_minus_ab': -8}". - print(out) - -When using ``outputs`` to request only a subset of a graph's outputs, GraphKit executes only the ``operation`` nodes in the graph that are on a path from the inputs to the requested outputs. For example, the ``abspow1`` operation will not be executed here. - -Short-circuiting a graph computation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can short-circuit a graph computation, making certain inputs unnecessary, by providing a value in the graph that is further downstream in the graph than those inputs. For example, in the graph we've been working with, you could provide the value of ``a_minus_ab`` to make the inputs ``a`` and ``b`` unnecessary:: - - # Run the graph and request a subset of the outputs. - out = graph({'a_minus_ab': -8}) - - # Prints "{'a_minus_ab': -8, 'abs_a_minus_ab_cubed': 512}". - print(out) - -When you do this, any ``operation`` nodes that are not on a path from the downstream input to the requested outputs (i.e. predecessors of the downstream input) are not computed. For example, the ``mul1`` and ``sub1`` operations are not executed here. - -This can be useful if you have a graph that accepts alternative forms of the same input. For example, if your graph requires a ``PIL.Image`` as input, you could allow your graph to be run in an API server by adding an earlier ``operation`` that accepts as input a string of raw image data and converts that data into the needed ``PIL.Image``. Then, you can either provide the raw image data string as input, or you can provide the ``PIL.Image`` if you have it and skip providing the image data string. - -Adding on to an existing computation graph ------------------------------------------- - -Sometimes you will have an existing computation graph to which you want to add operations. This is simple, since ``compose`` can compose whole graphs along with individual ``operation`` instances. For example, if we have ``graph`` as above, we can add another operation to it to create a new graph:: - - # Add another subtraction operation to the graph. - bigger_graph = compose(name="bigger_graph")( - graph, - operation(name="sub2", needs=["a_minus_ab", "c"], provides="a_minus_ab_minus_c")(sub) - ) - - # Run the graph and print the output. Prints "{'a_minus_ab_minus_c': -13}" - print(bigger_graph({'a': 2, 'b': 5, 'c': 5}, outputs=["a_minus_ab_minus_c"])) - -This yields a graph that looks like this: - -.. image:: images/bigger_example_graph.svg - - - -More complicated composition: merging computation graphs ----------------------------------------------------------- - -Sometimes you will have two computation graphs---perhaps ones that share operations---you want to combine into one. In the simple case, where the graphs don't share operations or where you don't care whether a duplicated operation is run multiple (redundant) times, you can just do something like this:: - - combined_graph = compose(name="combined_graph")(graph1, graph2) - -However, if you want to combine graphs that share operations and don't want to pay the price of running redundant computations, you can set the ``merge`` parameter of ``compose()`` to ``True``. This will consolidate redundant ``operation`` nodes (based on ``name``) into a single node. For example, let's say we have ``graph``, as in the examples above, along with this graph:: - - # This graph shares the "mul1" operation with graph. - another_graph = compose(name="another_graph")( - operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), - operation(name="mul2", needs=["c", "ab"], provides=["cab"])(mul) - ) - -We can merge ``graph`` and ``another_graph`` like so, avoiding a redundant ``mul1`` operation:: - - merged_graph = compose(name="merged_graph", merge=True)(graph, another_graph) - -This ``merged_graph`` will look like this: - -.. image:: images/example_merged_graph.svg - -As always, we can run computations with this graph by simply calling it:: - - # Prints "{'cab': 50}". - print(merged_graph({'a': 2, 'b': 5, 'c': 5}, outputs=["cab"])) diff --git a/docs/source/images/GraphkitFlowchart-v1.2.4.svg b/docs/source/images/GraphkitFlowchart-v1.2.4.svg new file mode 100644 index 00000000..2b22a63d --- /dev/null +++ b/docs/source/images/GraphkitFlowchart-v1.2.4.svg @@ -0,0 +1,80 @@ + + + + + + +%3 + +graphkit-v1.2.4 flowchart + + +operations + +operations + + + +compose + +compose & compile + + + +operations->compose + + + + + +network + +network + + + +compose->network + + + + + +compute + +compute + + + +network->compute + + + + + +data + +inputs & outputs + + + +data->compute + + + + + +solution + +solution + + + +compute->solution + + + + + diff --git a/docs/source/images/GraphkitFlowchart-v1.3.0.svg b/docs/source/images/GraphkitFlowchart-v1.3.0.svg new file mode 100644 index 00000000..ac88bfec --- /dev/null +++ b/docs/source/images/GraphkitFlowchart-v1.3.0.svg @@ -0,0 +1,145 @@ + + + + + + +%3 + +graphkit-v1.3.0 flowchart + +cluster_compute + +compute + + + +operations + +operations + + + +compose + +compose + + + +operations->compose + + + + + +network + +network + + + +compose->network + + + + + +compile + +compile + + + +network->compile + + + + + +inputs + +input names + + + +inputs->compile + + + + + +outputs + +output names + + + +outputs->compile + + + + + +plan + +execution plan + + + +compile->plan + + + + + +execute + +execute + + + +plan->execute + + + + + +solution + +solution + + + +execute->solution + + + + + +overwrites + +overwrites + + + +execute->overwrites + + + + + +values + +input values + + + +values->execute + + + + + diff --git a/docs/source/images/GraphkitLegend.svg b/docs/source/images/GraphkitLegend.svg new file mode 100644 index 00000000..11887496 --- /dev/null +++ b/docs/source/images/GraphkitLegend.svg @@ -0,0 +1,162 @@ + + + + + + +G + + +cluster_legend + +Graphkit Legend + + + +operation + +operation + + + +graphop + +graph operation + + + + +insteps + +execution step + + + + +executed + +executed + + + + +data + +data + + + +input + +input + + + + +output + +output + + + + +inp_out + +inp+out + + + + +evicted + +evicted + + + + +pinned + +pinned + + + + +evpin + +evict+pin + + + + +sol + +in solution + + + + + +e2 + +dependency + + + +e1->e2 + + + + + +e3 + +optional + + + +e2->e3 + + + + + +e33 + +sideffect + + + +e3->e33 + + + + + +e4 + +pruned dependency + + + +e33->e4 + + + + + +e5 + +execution sequence + + + +e4->e5 + + +1 + + + diff --git a/docs/source/images/intro.svg b/docs/source/images/intro.svg new file mode 100644 index 00000000..4469543f --- /dev/null +++ b/docs/source/images/intro.svg @@ -0,0 +1,143 @@ + + + + + + +G + +graphop + +cluster_after prunning + +after prunning + + + +abspow1 + +abspow1 + + + +abs_a_minus_ab_cubed + +abs_a_minus_ab_cubed + + + +abspow1->abs_a_minus_ab_cubed + + + + + +a + +a + + + +mul1 + +mul1 + + + +a->mul1 + + + + + +ab + +ab + + + +a->ab + + +4 + + + +sub1 + +sub1 + + + +a->sub1 + + + + + +b + +b + + + +mul1->b + + +1 + + + +mul1->ab + + + + + +b->mul1 + + + + + +b->sub1 + + +2 + + + +ab->sub1 + + + + + +sub1->a + + +3 + + + +a_minus_ab + +a_minus_ab + + + +sub1->a_minus_ab + + + + + +a_minus_ab->abspow1 + + + + + diff --git a/docs/source/images/sample_plot.svg b/docs/source/images/sample_plot.svg new file mode 100644 index 00000000..635e5e7c --- /dev/null +++ b/docs/source/images/sample_plot.svg @@ -0,0 +1,99 @@ + + + + + + +G + + + +a + +a + + + +must run + +must run + + + +a->must run + + + + + +overriden + +overriden + + + +must run->overriden + + + + + +must run->overriden + + +1 + + + +calced + +calced + + + +must run->calced + + + + + +add + +add + + + +overriden->add + + + + + +overriden->add + + +2 + + + +calced->add + + + + + +asked + +asked + + + +add->asked + + + + + diff --git a/docs/source/images/test_pruning_not_overrides_given_intermediate-asked.png b/docs/source/images/test_pruning_not_overrides_given_intermediate-asked.png new file mode 100644 index 00000000..c8e6cdb4 Binary files /dev/null and b/docs/source/images/test_pruning_not_overrides_given_intermediate-asked.png differ diff --git a/docs/source/index.rst b/docs/source/index.rst index 5c5e505c..34fa7f2a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,26 +8,31 @@ GraphKit ======== -.. image:: https://badge.fury.io/py/graphkit.svg - :target: https://badge.fury.io/py/graphkit -.. image:: https://travis-ci.org/yahoo/graphkit.svg?branch=master - :target: https://travis-ci.org/yahoo/graphkit -.. image:: https://codecov.io/gh/yahoo/graphkit/branch/master/graph/badge.svg - :target: https://codecov.io/gh/yahoo/graphkit +|travis-status| |cover-status| |gh-version| |pypi-version| |python-ver| +|dev-status| |downloads-count| |codestyle| |proj-lic| -**It's a DAG all the way down** +|gh-watch| |gh-star| |gh-fork| |gh-issues| + +**It's a DAG all the way down!** |sample-plot| + +Lightweight computation graphs for Python +----------------------------------------- + +GraphKit is an an understandable and lightweight Python module for building and running +ordered graphs of computations. +The API posits a fair compromise between features and complexity without precluding any. +It might be of use in computer vision, machine learning and other data science domains, +or become the core of a custom ETL pipelne. .. toctree:: :maxdepth: 2 operations - graph_composition - - -Lightweight computation graphs for Python ------------------------------------------ + composition + plotting + reference + changes -GraphKit is a lightweight Python module for creating and running ordered graphs of computations, where the nodes of the graph correspond to computational operations, and the edges correspond to output --> input dependencies between those operations. Such graphs are useful in computer vision, machine learning, and many other domains. .. _quick-start: @@ -38,6 +43,12 @@ Here's how to install:: pip install graphkit +OR with dependencies for plotting support (and you need to install `Graphviz +`_ program separately with your OS tools):: + + pip install graphkit[plot] + + Here's a Python script with an example GraphKit computation graph that produces multiple outputs (``a * b``, ``a - a * b``, and ``abs(a - a * b) ** 3``):: from operator import mul, sub @@ -49,27 +60,86 @@ Here's a Python script with an example GraphKit computation graph that produces return c # Compose the mul, sub, and abspow operations into a computation graph. - graph = compose(name="graph")( + graphop = compose(name="graphop")( operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), operation(name="abspow1", needs=["a_minus_ab"], provides=["abs_a_minus_ab_cubed"], params={"p": 3})(abspow) ) - # Run the graph and request all of the outputs. - out = graph({'a': 2, 'b': 5}) + # Run the graph-operation and request all of the outputs. + out = graphop({'a': 2, 'b': 5}) # Prints "{'a': 2, 'a_minus_ab': -8, 'b': 5, 'ab': 10, 'abs_a_minus_ab_cubed': 512}". print(out) - # Run the graph and request a subset of the outputs. - out = graph({'a': 2, 'b': 5}, outputs=["a_minus_ab"]) + # Run the graph-operation and request a subset of the outputs. + out = graphop({'a': 2, 'b': 5}, outputs=["a_minus_ab"]) # Prints "{'a_minus_ab': -8}". print(out) -As you can see, any function can be used as an operation in GraphKit, even ones imported from system modules! +As you can see, any function can be used as an operation in GraphKit, +even ones imported from system modules! + License ------- Code licensed under the Apache License, Version 2.0 license. See LICENSE file for terms. + + +.. |travis-status| image:: https://travis-ci.org/yahoo/graphkit.svg + :alt: Travis continuous integration testing ok? (Linux) + :target: https://travis-ci.org/yahoo/graphkit/builds + +.. |cover-status| image:: https://codecov.io/gh/yahoo/graphkit/branch/master/graph/badge.svg + :target: https://codecov.io/gh/yahoo/graphkit + +.. |gh-version| image:: https://img.shields.io/github/v/release/yahoo/graphkit.svg?label=GitHub%20release&include_prereleases + :target: https://github.com/yahoo/graphkit/releases + :alt: Latest version in GitHub + +.. |pypi-version| image:: https://img.shields.io/pypi/v/graphkit.svg?label=PyPi%20version + :target: https://pypi.python.org/pypi/graphkit/ + :alt: Latest version in PyPI + +.. |python-ver| image:: https://img.shields.io/pypi/pyversions/graphkit.svg?label=Python + :target: https://pypi.python.org/pypi/graphkit/ + :alt: Supported Python versions of latest release in PyPi + +.. |dev-status| image:: https://pypip.in/status/graphkit/badge.svg + :target: https://pypi.python.org/pypi/graphkit/ + :alt: Development Status + +.. |downloads-count| image:: https://pypip.in/download/graphkit/badge.svg?period=month&label=PyPi%20downloads + :target: https://pypi.python.org/pypi/graphkit/ + :alt: PyPi downloads + +.. |codestyle| image:: https://img.shields.io/badge/code%20style-black-black.svg + :target: https://github.com/ambv/black + :alt: Code Style + +.. |gh-watch| image:: https://img.shields.io/github/watchers/yahoo/graphkit.svg?style=social + :target: https://github.com/yahoo/graphkit + :alt: Github watchers + +.. |gh-star| image:: https://img.shields.io/github/stars/yahoo/graphkit.svg?style=social + :target: https://github.com/yahoo/graphkit + :alt: Github stargazers + +.. |gh-fork| image:: https://img.shields.io/github/forks/yahoo/graphkit.svg?style=social + :target: https://github.com/yahoo/graphkit + :alt: Github forks + +.. |gh-issues| image:: http://img.shields.io/github/issues/yahoo/graphkit.svg?style=social + :target: https://github.com/yahoo/graphkit/issues + :alt: Issues count + +.. |proj-lic| image:: https://img.shields.io/pypi/l/graphkit.svg + :target: https://www.apache.org/licenses/LICENSE-2.0 + :alt: Apache License, version 2.0 + +.. |sample-plot| image:: images/sample_plot.svg + :alt: sample graphkit plot + :width: 120px + :align: middle diff --git a/docs/source/operations.rst b/docs/source/operations.rst index b7b4dbad..55a84ebd 100644 --- a/docs/source/operations.rst +++ b/docs/source/operations.rst @@ -13,6 +13,7 @@ There are many ways to instantiate an ``operation``, and we'll get into more det .. autoclass:: graphkit.operation :members: __init__, __call__ :member-order: bysource + :special-members: Operations are just functions @@ -20,13 +21,13 @@ Operations are just functions At the heart of each ``operation`` is just a function, any arbitrary function. Indeed, you can instantiate an ``operation`` with a function and then call it just like the original function, e.g.:: - from operator import add - from graphkit import operation + >>> from operator import add + >>> from graphkit import operation - add_op = operation(name='add_op', needs=['a', 'b'], provides=['a_plus_b'])(add) + >>> add_op = operation(name='add_op', needs=['a', 'b'], provides=['a_plus_b'])(add) - # Passes! - assert add_op(3, 4) == add(3, 4) + >>> add_op(3, 4) == add(3, 4) + True Specifying graph structure: ``provides`` and ``needs`` @@ -42,24 +43,27 @@ When many operations are composed into a computation graph (see :ref:`graph-comp Let's look again at the operations from the script in :ref:`quick-start`, for example:: - from operator import mul, sub - from graphkit import compose, operation + >>> from operator import mul, sub + >>> from graphkit import compose, operation - # Computes |a|^p. - def abspow(a, p): - c = abs(a) ** p - return c + >>> # Computes |a|^p. + >>> def abspow(a, p): + ... c = abs(a) ** p + ... return c - # Compose the mul, sub, and abspow operations into a computation graph. - graph = compose(name="graph")( - operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), - operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), - operation(name="abspow1", needs=["a_minus_ab"], provides=["abs_a_minus_ab_cubed"], params={"p": 3})(abspow) - ) + >>> # Compose the mul, sub, and abspow operations into a computation graph. + >>> graphop = compose(name="graphop")( + ... operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), + ... operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), + ... operation(name="abspow1", needs=["a_minus_ab"], provides=["abs_a_minus_ab_cubed"], params={"p": 3})(abspow) + ... ) -The ``needs`` and ``provides`` arguments to the operations in this script define a computation graph that looks like this (where the circles are operations, squares are data, and octagons are parameters): +The ``needs`` and ``provides`` arguments to the operations in this script define a computation graph that looks like this (where the oval are operations, squares/houses are data): -.. image:: images/example_graph.svg +.. image:: images/intro.svg + +.. Tip:: + See :ref:`plotting` on how to make diagrams like this. Constant operation parameters: ``params`` @@ -80,38 +84,38 @@ Decorator specification If you are defining your computation graph and the functions that comprise it all in the same script, the decorator specification of ``operation`` instances might be particularly useful, as it allows you to assign computation graph structure to functions as they are defined. Here's an example:: - from graphkit import operation, compose + >>> from graphkit import operation, compose - @operation(name='foo_op', needs=['a', 'b', 'c'], provides='foo') - def foo(a, b, c): - return c * (a + b) + >>> @operation(name='foo_op', needs=['a', 'b', 'c'], provides='foo') + ... def foo(a, b, c): + ... return c * (a + b) - graph = compose(name='foo_graph')(foo) + >>> graphop = compose(name='foo_graph')(foo) Functional specification ^^^^^^^^^^^^^^^^^^^^^^^^ If the functions underlying your computation graph operations are defined elsewhere than the script in which your graph itself is defined (e.g. they are defined in another module, or they are system functions), you can use the functional specification of ``operation`` instances:: - from operator import add, mul - from graphkit import operation, compose + >>> from operator import add, mul + >>> from graphkit import operation, compose - add_op = operation(name='add_op', needs=['a', 'b'], provides='sum')(add) - mul_op = operation(name='mul_op', needs=['c', 'sum'], provides='product')(mul) + >>> add_op = operation(name='add_op', needs=['a', 'b'], provides='sum')(add) + >>> mul_op = operation(name='mul_op', needs=['c', 'sum'], provides='product')(mul) - graph = compose(name='add_mul_graph')(add_op, mul_op) + >>> graphop = compose(name='add_mul_graph')(add_op, mul_op) The functional specification is also useful if you want to create multiple ``operation`` instances from the same function, perhaps with different parameter values, e.g.:: - from graphkit import operation, compose + >>> from graphkit import operation, compose - def mypow(a, p=2): - return a ** p + >>> def mypow(a, p=2): + ... return a ** p - pow_op1 = operation(name='pow_op1', needs=['a'], provides='a_squared')(mypow) - pow_op2 = operation(name='pow_op2', needs=['a'], params={'p': 3}, provides='a_cubed')(mypow) + >>> pow_op1 = operation(name='pow_op1', needs=['a'], provides='a_squared')(mypow) + >>> pow_op2 = operation(name='pow_op2', needs=['a'], params={'p': 3}, provides='a_cubed')(mypow) - graph = compose(name='two_pows_graph')(pow_op1, pow_op2) + >>> graphop = compose(name='two_pows_graph')(pow_op1, pow_op2) A slightly different approach can be used here to accomplish the same effect by creating an operation "factory":: @@ -125,7 +129,7 @@ A slightly different approach can be used here to accomplish the same effect by pow_op1 = pow_op_factory(name='pow_op1', needs=['a'], provides='a_squared') pow_op2 = pow_op_factory(name='pow_op2', needs=['a'], params={'p': 3}, provides='a_cubed') - graph = compose(name='two_pows_graph')(pow_op1, pow_op2) + graphop = compose(name='two_pows_graph')(pow_op1, pow_op2) Modifiers on ``operation`` inputs and outputs @@ -134,3 +138,4 @@ Modifiers on ``operation`` inputs and outputs Certain modifiers are available to apply to input or output values in ``needs`` and ``provides``, for example to designate an optional input. These modifiers are available in the ``graphkit.modifiers`` module: .. autoclass:: graphkit.modifiers.optional +.. autoclass:: graphkit.modifiers.sideffect diff --git a/docs/source/plotting.rst b/docs/source/plotting.rst new file mode 100644 index 00000000..ebb4e90a --- /dev/null +++ b/docs/source/plotting.rst @@ -0,0 +1,120 @@ +###################### +Plotting and Debugging +###################### + +.. _plotting: + +Plotting +-------- + +For :ref:`debugging` it is necessary to visualize the graph-operation. +You may plot the original plot and annotate on top the *execution plan* and +solution of the last computation, calling methods with arguments like this:: + + graphop.plot(show=True) # open a matplotlib window + graphop.plot("intro.svg") # other supported formats: png, jpg, pdf, ... + graphop.plot() # without arguments return a pydot.DOT object + graphop.plot(solution=out) # annotate graph with solution values + +.. image:: images/intro.svg + :alt: Intro graph + +.. figure:: images/GraphkitLegend.svg + :alt: Graphkit Legend + + The legend for all graphkit diagrams, generated by :func:`graphkit.plot.legend()`. + +The same ``plot()`` methods are defined on a :class:`NetworkOperation`, +:class:`Network` & :class:`ExecutionPlan`, each one capable to produce diagrams +with increasing complexity. Whenever possible, the top-level ``plot()`` methods +delegates to the ones below. + +For instance, when a net-operation has just been composed, plotting it will +come out bare bone, with just the 2 types of nodes (data & operations), their +dependencies, and the sequence of the execution-plan. + +But as soon as you run it, the net plot calls will print more of the internals. +These are based on the ``graph_op.net.last_plan`` attribute which *caches* +the last run to inspect it. If you want the bare-bone diagram, simply reset it:: + + netop.net.last_plan = None + +.. Note:: + For plots, `Graphviz `_ program must be in your PATH, + and ``pydot`` & ``matplotlib`` python packages installed. + You may install both when installing ``graphkit`` with its ``plot`` extras:: + + pip install graphkit[plot] + +.. Tip:: + The `pydot.Dot `_ instances returned by ``plot()`` + are rendered directly in *Jupyter/IPython* notebooks as SVG images. + + +.. _debugging: + +Errors & debugging +------------------ + +If an operation fails, the original exception gets annotated +with the folllowing properties, as a debug aid: + +>>> from pprint import pprint + +>>> def scream(*args): +... raise ValueError("Wrong!") + +>>> try: +... compose("errgraph")( +... operation(name="screamer", needs=['a'], provides=["foo"])(scream) +... )({'a': None}) +... except ValueError as ex: +... pprint(ex.graphkit_aid) +{'network': + ... + 'operation': FunctionalOperation(name='screamer', needs=['a'], provides=['foo']), + 'operation_args': {'args': [None], 'kwargs': {}}, + 'operation_fnouts': None, + 'operation_outs': None, + 'operation_results': None, + 'plan': ExecutionPlan(inputs=('a',), outputs=(), steps: + +--FunctionalOperation(name='screamer', needs=['a'], provides=['foo'])), + 'solution': {'a': None}} + + +The following annotated attributes might have values on an exception ``ex``: + +``ex.network`` + the innermost network owning the failed operation/function + +``ex.plan`` + the innermost plan that executing when a operation crashed + +``ex.operation`` + the innermost operation that failed + +``ex.operation_args`` + either a 2-tuple ``(args, kwargs)`` or just the ``args`` fed to the operation + +``ex.operation_fnouts`` + the names of the outputs the function was expected to return + +``ex.operation_outs`` + the names eventually the graph needed from the operation + (a subset of the above) + +``ex.operation_results`` + the values dict, if any; it maybe a *zip* of the provides + with the actual returned values of the function, ot the raw results. + +.. note:: + The :ref:`plotting` capabilities, along with the above annotation of exceptions + with the internal state of plan/operation often renders a debugger session + unnecessary. But since the state of the annotated values might be incomple, + you may not always avoid one. + + +Execution internals +------------------- +.. automodule:: graphkit.network + :noindex: diff --git a/docs/source/reference.rst b/docs/source/reference.rst new file mode 100644 index 00000000..a64b29c7 --- /dev/null +++ b/docs/source/reference.rst @@ -0,0 +1,31 @@ +============= +API Reference +============= + +Module: `base` +============== + +.. automodule:: graphkit.base + :members: + :undoc-members: + +Module: `functional` +==================== + +.. automodule:: graphkit.functional + :members: + :undoc-members: + +Module: `network` +================= + +.. automodule:: graphkit.network + :members: + :undoc-members: + +Module: `plot` +============== + +.. automodule:: graphkit.plot + :members: + :undoc-members: diff --git a/graphkit/__init__.py b/graphkit/__init__.py index b930a65c..0ee2bfdc 100644 --- a/graphkit/__init__.py +++ b/graphkit/__init__.py @@ -1,10 +1,16 @@ # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. +"""Lightweight computation graphs for Python.""" -__author__ = 'hnguyen' -__version__ = '1.2.4' +__author__ = "hnguyen" +__version__ = "1.3.0" +__license__ = "Apache-2.0" +__title__ = "graphkit" +__summary__ = __doc__.splitlines()[0] +__uri__ = "https://github.com/yahoo/graphkit" from .functional import operation, compose +from .modifiers import * # noqa, on purpose to include any new modifiers # For backwards compatibility from .base import Operation diff --git a/graphkit/base.py b/graphkit/base.py index 1c04e8d5..b50a64e6 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -1,5 +1,12 @@ # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. +try: + from collections import abc +except ImportError: + import collections as abc + +from . import plot + class Data(object): """ @@ -10,6 +17,7 @@ class Data(object): This class an "abstract" class that should be extended by any class working with data in the HiC framework. """ + def __init__(self, **kwargs): pass @@ -19,6 +27,7 @@ def get_data(self): def set_data(self, data): raise NotImplementedError + class Operation(object): """ This is an abstract class representing a data transformation. To use this, @@ -33,24 +42,28 @@ def __init__(self, **kwargs): important when connecting layers and data in a Network object, as the names are used to construct the graph. - :param str name: The name the operation (e.g. conv1, conv2, etc..) + :param str name: + The name the operation (e.g. conv1, conv2, etc..) - :param list needs: Names of input data objects this layer requires. + :param list needs: + Names of input data objects this layer requires. - :param list provides: Names of output data objects this provides. + :param list provides: + Names of output data objects this provides. - :param dict params: A dict of key/value pairs representing parameters - associated with your operation. These values will be - accessible using the ``.params`` attribute of your object. - NOTE: It's important that any values stored in this - argument must be pickelable. + :param dict params: + A dict of key/value pairs representing parameters + associated with your operation. These values will be + accessible using the ``.params`` attribute of your object. + NOTE: It's important that any values stored in this + argument must be pickelable. """ # (Optional) names for this layer, and the data it needs and provides - self.name = kwargs.get('name') - self.needs = kwargs.get('needs') - self.provides = kwargs.get('provides') - self.params = kwargs.get('params', {}) + self.name = kwargs.get("name") + self.needs = kwargs.get("needs") + self.provides = kwargs.get("provides") + self.params = kwargs.get("params", {}) # call _after_init as final step of initialization self._after_init() @@ -60,8 +73,7 @@ def __eq__(self, other): Operation equality is based on name of layer. (__eq__ and __hash__ must be overridden together) """ - return bool(self.name is not None and - self.name == getattr(other, 'name', None)) + return bool(self.name is not None and self.name == getattr(other, "name", None)) def __hash__(self): """ @@ -74,6 +86,7 @@ def compute(self, inputs): """ This method must be implemented to perform this layer's feed-forward computation on a given set of inputs. + :param list inputs: A list of :class:`Data` objects on which to run the layer's feed-forward computation. @@ -83,18 +96,32 @@ def compute(self, inputs): ``inputs``. """ - raise NotImplementedError + raise NotImplementedError("Define callable of %r!" % self) def _compute(self, named_inputs, outputs=None): - inputs = [named_inputs[d] for d in self.needs] - results = self.compute(inputs) - - results = zip(self.provides, results) - if outputs: - outputs = set(outputs) - results = filter(lambda x: x[0] in outputs, results) - - return dict(results) + try: + args = [named_inputs[d] for d in self.needs] + results = self.compute(args) + + results = zip(self.provides, results) + + if outputs: + outs = set(outputs) + results = filter(lambda x: x[0] in outs, results) + + return dict(results) + except Exception as ex: + ## Annotate exception with debugging aid on errors. + # + locs = locals() + err_aid = getattr(ex, "graphkit_aid", {}) + err_aid.setdefault("operation", self) + err_aid.setdefault("operation_args", locs.get("args")) + err_aid.setdefault("operation_fnouts", locs.get("outputs")) + err_aid.setdefault("operation_outs", locs.get("outputs")) + err_aid.setdefault("operation_results", locs.get("results")) + setattr(ex, "graphkit_aid", err_aid) + raise def _after_init(self): """ @@ -117,11 +144,11 @@ def __getstate__(self): result = {} # this check should get deprecated soon. its for downward compatibility # with earlier pickled operation objects - if hasattr(self, 'params'): - result["params"] = self.__dict__['params'] - result["needs"] = self.__dict__['needs'] - result["provides"] = self.__dict__['provides'] - result["name"] = self.__dict__['name'] + if hasattr(self, "params"): + result["params"] = self.__dict__["params"] + result["needs"] = self.__dict__["needs"] + result["provides"] = self.__dict__["provides"] + result["name"] = self.__dict__["name"] return result @@ -137,44 +164,83 @@ def __repr__(self): """ Display more informative names for the Operation class """ - return u"%s(name='%s', needs=%s, provides=%s)" % \ - (self.__class__.__name__, - self.name, - self.needs, - self.provides) + def aslist(i): + if i and not isinstance(i, str): + return list(i) + return i -class NetworkOperation(Operation): + return u"%s(name='%s', needs=%s, provides=%s)" % ( + self.__class__.__name__, + getattr(self, "name", None), + aslist(getattr(self, "needs", None)), + aslist(getattr(self, "provides", None)), + ) + + +class NetworkOperation(Operation, plot.Plotter): def __init__(self, **kwargs): - self.net = kwargs.pop('net') + self.net = kwargs.pop("net") Operation.__init__(self, **kwargs) # set execution mode to single-threaded sequential by default self._execution_method = "sequential" + self._overwrites_collector = None + + def _build_pydot(self, **kws): + """delegate to network""" + kws.setdefault("title", self.name) + plotter = self.net.last_plan or self.net + return plotter._build_pydot(**kws) def _compute(self, named_inputs, outputs=None): - return self.net.compute(outputs, named_inputs, method=self._execution_method) + return self.net.compute( + named_inputs, + outputs, + method=self._execution_method, + overwrites_collector=self._overwrites_collector, + ) def __call__(self, *args, **kwargs): return self._compute(*args, **kwargs) + def compile(self, *args, **kwargs): + return self.net.compile(*args, **kwargs) + def set_execution_method(self, method): """ Determine how the network will be executed. - Args: - method: str - If "parallel", execute graph operations concurrently - using a threadpool. + :param str method: + If "parallel", execute graph operations concurrently + using a threadpool. """ - options = ['parallel', 'sequential'] - assert method in options + choices = ["parallel", "sequential"] + if method not in choices: + raise ValueError( + "Invalid computation method %r! Must be one of %s" % (method, choices) + ) self._execution_method = method - def plot(self, filename=None, show=False): - self.net.plot(filename=filename, show=show) + def set_overwrites_collector(self, collector): + """ + Asks to put all *overwrites* into the `collector` after computing + + An "overwrites" is intermediate value calculated but NOT stored + into the results, becaues it has been given also as an intemediate + input value, and the operation that would overwrite it MUST run for + its other results. + + :param collector: + a mutable dict to be fillwed with named values + """ + if collector is not None and not isinstance(collector, abc.MutableMapping): + raise ValueError( + "Overwrites collector was not a MutableMapping, but: %r" % collector + ) + self._overwrites_collector = collector def __getstate__(self): state = Operation.__getstate__(self) - state['net'] = self.__dict__['net'] + state["net"] = self.__dict__["net"] return state diff --git a/graphkit/functional.py b/graphkit/functional.py index 65388973..b5c22f02 100644 --- a/graphkit/functional.py +++ b/graphkit/functional.py @@ -1,44 +1,77 @@ # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. +import networkx as nx +from boltons.setutils import IndexedSet as iset -from itertools import chain - -from .base import Operation, NetworkOperation +from .base import NetworkOperation, Operation +from .modifiers import optional, sideffect from .network import Network -from .modifiers import optional class FunctionalOperation(Operation): def __init__(self, **kwargs): - self.fn = kwargs.pop('fn') + self.fn = kwargs.pop("fn") Operation.__init__(self, **kwargs) def _compute(self, named_inputs, outputs=None): - inputs = [named_inputs[d] for d in self.needs if not isinstance(d, optional)] - - # Find any optional inputs in named_inputs. Get only the ones that - # are present there, no extra `None`s. - optionals = {n: named_inputs[n] for n in self.needs if isinstance(n, optional) and n in named_inputs} - - # Combine params and optionals into one big glob of keyword arguments. - kwargs = {k: v for d in (self.params, optionals) for k, v in d.items()} - result = self.fn(*inputs, **kwargs) - if len(self.provides) == 1: - result = [result] - - result = zip(self.provides, result) - if outputs: - outputs = set(outputs) - result = filter(lambda x: x[0] in outputs, result) - - return dict(result) + try: + args = [ + named_inputs[n] + for n in self.needs + if not isinstance(n, optional) and not isinstance(n, sideffect) + ] + + # Find any optional inputs in named_inputs. Get only the ones that + # are present there, no extra `None`s. + optionals = { + n: named_inputs[n] + for n in self.needs + if isinstance(n, optional) and n in named_inputs + } + + # Combine params and optionals into one big glob of keyword arguments. + kwargs = {k: v for d in (self.params, optionals) for k, v in d.items()} + + # Don't expect sideffect outputs. + provides = [n for n in self.provides if not isinstance(n, sideffect)] + + result = self.fn(*args, **kwargs) + + if not provides: + # All outputs were sideffects. + return {} + + if len(provides) == 1: + result = [result] + + result = zip(provides, result) + if outputs: + outputs = set(n for n in outputs if not isinstance(n, sideffect)) + result = filter(lambda x: x[0] in outputs, result) + + return dict(result) + except Exception as ex: + ## Annotate exception with debugging aid on errors. + # + locs = locals() + err_aid = getattr(ex, "graphkit_aid", {}) + err_aid.setdefault("operation", self) + err_aid.setdefault( + "operation_args", + {"args": locs.get("args"), "kwargs": locs.get("kwargs")}, + ) + err_aid.setdefault("operation_fnouts", locs.get("outputs")) + err_aid.setdefault("operation_outs", locs.get("outputs")) + err_aid.setdefault("operation_results", locs.get("results")) + setattr(ex, "graphkit_aid", err_aid) + raise def __call__(self, *args, **kwargs): return self.fn(*args, **kwargs) def __getstate__(self): state = Operation.__getstate__(self) - state['fn'] = self.__dict__['fn'] + state["fn"] = self.__dict__["fn"] return state @@ -66,7 +99,7 @@ class operation(Operation): :param dict params: A dict of key/value pairs representing constant parameters associated with your operation. These can correspond to either - ``args`` or ``kwargs`` of ``fn`. + ``args`` or ``kwargs`` of ``fn``. """ def __init__(self, fn=None, **kwargs): @@ -76,22 +109,26 @@ def __init__(self, fn=None, **kwargs): def _normalize_kwargs(self, kwargs): # Allow single value for needs parameter - if 'needs' in kwargs and type(kwargs['needs']) == str: - assert kwargs['needs'], "empty string provided for `needs` parameters" - kwargs['needs'] = [kwargs['needs']] + needs = kwargs["needs"] + if isinstance(needs, str) and not isinstance(needs, optional): + assert needs, "empty string provided for `needs` parameters" + kwargs["needs"] = [needs] # Allow single value for provides parameter - if 'provides' in kwargs and type(kwargs['provides']) == str: - assert kwargs['provides'], "empty string provided for `needs` parameters" - kwargs['provides'] = [kwargs['provides']] + provides = kwargs.get("provides") + if isinstance(provides, str): + assert provides, "empty string provided for `needs` parameters" + kwargs["provides"] = [provides] - assert kwargs['name'], "operation needs a name" - assert type(kwargs['needs']) == list, "no `needs` parameter provided" - assert type(kwargs['provides']) == list, "no `provides` parameter provided" - assert hasattr(kwargs['fn'], '__call__'), "operation was not provided with a callable" + assert kwargs["name"], "operation needs a name" + assert isinstance(kwargs["needs"], list), "no `needs` parameter provided" + assert isinstance(kwargs["provides"], list), "no `provides` parameter provided" + assert hasattr( + kwargs["fn"], "__call__" + ), "operation was not provided with a callable" - if type(kwargs['params']) is not dict: - kwargs['params'] = {} + if type(kwargs["params"]) is not dict: + kwargs["params"] = {} return kwargs @@ -132,14 +169,21 @@ def __repr__(self): """ Display more informative names for the Operation class """ - return u"%s(name='%s', needs=%s, provides=%s, fn=%s)" % \ - (self.__class__.__name__, - self.name, - self.needs, - self.provides, - self.fn.__name__) + def aslist(i): + if i and not isinstance(i, str): + return list(i) + return i + func_name = getattr(self, "fn") + func_name = func_name and getattr(func_name, "__name__", None) + return u"%s(name='%s', needs=%s, provides=%s, fn=%s)" % ( + self.__class__.__name__, + getattr(self, "name", None), + aslist(getattr(self, "needs", None)), + aslist(getattr(self, "provides", None)), + func_name, + ) class compose(object): @@ -185,27 +229,25 @@ def __call__(self, *operations): # If merge is desired, deduplicate operations before building network if self.merge: - merge_set = set() + merge_set = iset() # Preseve given node order. for op in operations: if isinstance(op, NetworkOperation): - net_ops = filter(lambda x: isinstance(x, Operation), op.net.steps) - merge_set.update(net_ops) + netop_nodes = nx.topological_sort(op.net.graph) + merge_set.update(s for s in netop_nodes if isinstance(s, Operation)) else: merge_set.add(op) - operations = list(merge_set) - - def order_preserving_uniquifier(seq, seen=None): - seen = seen if seen else set() - seen_add = seen.add - return [x for x in seq if not (x in seen or seen_add(x))] + operations = merge_set - provides = order_preserving_uniquifier(chain(*[op.provides for op in operations])) - needs = order_preserving_uniquifier(chain(*[op.needs for op in operations]), set(provides)) + provides = iset(p for op in operations for p in op.provides) + # Mark them all as optional, now that #18 calmly ignores + # non-fully satisfied operations. + needs = iset(optional(n) for op in operations for n in op.needs) - provides - # compile network + # Build network net = Network() for op in operations: net.add_op(op) - net.compile() - return NetworkOperation(name=self.name, needs=needs, provides=provides, params={}, net=net) + return NetworkOperation( + name=self.name, needs=needs, provides=provides, params={}, net=net + ) diff --git a/graphkit/modifiers.py b/graphkit/modifiers.py index 652a7969..e38c2615 100644 --- a/graphkit/modifiers.py +++ b/graphkit/modifiers.py @@ -8,6 +8,7 @@ file associated with the project for terms. """ + class optional(str): """ Input values in ``needs`` may be designated as optional using this modifier. @@ -19,21 +20,77 @@ class optional(str): Here is an example of an operation that uses an optional argument:: - from graphkit import operation, compose - from graphkit.modifiers import optional + >>> from graphkit import operation, compose, optional + + >>> # Function that adds either two or three numbers. + >>> def myadd(a, b, c=0): + ... return a + b + c + + >>> # Designate c as an optional argument. + >>> graph = compose('mygraph')( + ... operation(name='myadd', needs=['a', 'b', optional('c')], provides='sum')(myadd) + ... ) + >>> graph + NetworkOperation(name='mygraph', + needs=[optional('a'), optional('b'), optional('c')], + provides=['sum']) + + >>> # The graph works with and without 'c' provided as input. + >>> graph({'a': 5, 'b': 2, 'c': 4})['sum'] + 11 + >>> graph({'a': 5, 'b': 2}) + {'a': 5, 'b': 2, 'sum': 7} + + """ + + __slots__ = () # avoid __dict__ on instances + + def __repr__(self): + return "optional('%s')" % self + - # Function that adds either two or three numbers. - def myadd(a, b, c=0): - return a + b + c +class sideffect(str): + """ + Inputs & outputs in ``needs`` & ``provides`` may be designated as *sideffects* + using this modifier. *Tokens* work as usual while solving the DAG but + they are never assigned any values to/from the ``operation`` functions. + Specifically: + + - input sideffects are NOT fed into the function; + - output sideffects are NOT expected from the function. + + Their purpose is to describe functions that have modify internal state + their arguments ("side-effects"). + Note that an ``operation`` with just a single *sideffect* output return + no value at all, but it would still be called for its side-effects only. + + A typical use case is to signify columns required to produce new ones in + pandas dataframes:: - # Designate c as an optional argument. - graph = compose('mygraph')( - operator(name='myadd', needs=['a', 'b', optional('c')], provides='sum')(myadd) - ) + >>> from graphkit import operation, compose, sideffect - # The graph works with and without 'c' provided as input. - assert graph({'a': 5, 'b': 2, 'c': 4})['sum'] == 11 - assert graph({'a': 5, 'b': 2})['sum'] == 7 + >>> # Function appending a new dataframe column from two pre-existing ones. + >>> def addcolumns(df): + ... df['sum'] = df['a'] + df['b'] + + >>> # Designate `a`, `b` & `sum` column names as an sideffect arguments. + >>> graph = compose('mygraph')( + ... operation( + ... name='addcolumns', + ... needs=['df', sideffect('a'), sideffect('b')], + ... provides=[sideffect('sum')])(addcolumns) + ... ) + >>> graph + NetworkOperation(name='mygraph', needs=[optional('df'), optional('a'), optional('b')], provides=[sideffect('sum')]) + + >>> # The graph works with and without 'c' provided as input. + >>> df = pd.DataFrame({'a': [5], 'b': [2]}) # doctest: +SKIP + >>> graph({'df': df})['sum'] == 11 # doctest: +SKIP + True """ - pass + + __slots__ = () # avoid __dict__ on instances + + def __repr__(self): + return "sideffect('%s')" % self diff --git a/graphkit/network.py b/graphkit/network.py index 0df3ddf8..02abd9a7 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -1,66 +1,174 @@ # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. - -import time +""" +Network-based computation of operations & data. + +The execution of network *operations* is splitted in 2 phases: + +COMPILE: + prune unsatisfied nodes, sort dag topologically & solve it, and + derive the *execution steps* (see below) based on the given *inputs* + and asked *outputs*. + +EXECUTE: + sequential or parallel invocation of the underlying functions + of the operations with arguments from the ``solution``. + +Computations are based on 5 data-structures: + +:attr:`Network.graph` + A ``networkx`` graph (yet a DAG) containing interchanging layers of + :class:`Operation` and :class:`DataPlaceholderNode` nodes. + They are layed out and connected by repeated calls of + :meth:`~Network.add_OP`. + + The computation starts with :meth:`~Network._prune_graph()` extracting + a *DAG subgraph* by *pruning* its nodes based on given inputs and + requested outputs in :meth:`~Network.compute()`. + +:attr:`ExecutionPlan.dag` + An directed-acyclic-graph containing the *pruned* nodes as build by + :meth:`~Network._prune_graph()`. This pruned subgraph is used to decide + the :attr:`ExecutionPlan.steps` (below). + The containing :class:`ExecutionPlan.steps` instance is cached + in :attr:`_cached_plans` across runs with inputs/outputs as key. + +:attr:`ExecutionPlan.steps` + It is the list of the operation-nodes only + from the dag (above), topologically sorted, and interspersed with + *instructions steps* needed to complete the run. + It is built by :meth:`~Network._build_execution_steps()` based on + the subgraph dag extracted above. + The containing :class:`ExecutionPlan.steps` instance is cached + in :attr:`_cached_plans` across runs with inputs/outputs as key. + + The *instructions* items achieve the following: + + - :class:`DeleteInstruction`: delete items from `solution` as soon as + they are not needed further down the dag, to reduce memory footprint + while computing. + + - :class:`PinInstruction`: avoid overwritting any given intermediate + inputs, and still allow their providing operations to run + (because they are needed for their other outputs). + +:var solution: + a local-var in :meth:`~Network.compute()`, initialized on each run + to hold the values of the given inputs, generated (intermediate) data, + and output values. + It is returned as is if no specific outputs requested; no data-eviction + happens then. + +:arg overwrites: + The optional argument given to :meth:`~Network.compute()` to colect the + intermediate *calculated* values that are overwritten by intermediate + (aka "pinned") input-values. +""" +import logging import os -import networkx as nx - +import sys +import time +from collections import defaultdict, namedtuple from io import StringIO +from itertools import chain + +import networkx as nx +from boltons.setutils import IndexedSet as iset +from networkx import DiGraph +from . import plot from .base import Operation +from .modifiers import optional, sideffect + +log = logging.getLogger(__name__) + + +if sys.version_info < (3, 6): + """ + Consistently ordered variant of :class:`~networkx.DiGraph`. + + PY3.6 has inmsertion-order dicts, but PY3.5 has not. + And behvavior *and TCs) in these environments may fail spuriously! + Still *subgraphs* may not patch! + + Fix from: + https://networkx.github.io/documentation/latest/reference/classes/ordered.html#module-networkx.classes.ordered + """ + from networkx import OrderedDiGraph as DiGraph class DataPlaceholderNode(str): """ - A node for the Network graph that describes the name of a Data instance - produced or required by a layer. + Dag node naming a data-value produced or required by an operation. """ + def __repr__(self): return 'DataPlaceholderNode("%s")' % self class DeleteInstruction(str): """ - An instruction for the compiled list of evaluation steps to free or delete - a Data instance from the Network's cache after it is no longer needed. + Execution step to delete a computed value from the `solution`. + + It's a step in :attr:`ExecutionPlan.steps` for the data-node `str` that + frees its data-value from `solution` after it is no longer needed, + to reduce memory footprint while computing the graph. """ + def __repr__(self): return 'DeleteInstruction("%s")' % self -class Network(object): +class PinInstruction(str): """ - This is the main network implementation. The class contains all of the - code necessary to weave together operations into a directed-acyclic-graph (DAG) - and pass data through. + Execution step to replace a computed value in the `solution` from the inputs, + + and to store the computed one in the ``overwrites`` instead + (both `solution` & ``overwrites`` are local-vars in :meth:`~Network.compute()`). + + It's a step in :attr:`ExecutionPlan.steps` for the data-node `str` that + ensures the corresponding intermediate input-value is not overwritten when + its providing function(s) could not be pruned, because their other outputs + are needed elesewhere. """ - def __init__(self, **kwargs): - """ - """ + def __repr__(self): + return 'PinInstruction("%s")' % self + + +class Network(plot.Plotter): + """ + Assemble operations & data into a directed-acyclic-graph (DAG) to run them. + + """ + def __init__(self, **kwargs): # directed graph of layer instances and data-names defining the net. - self.graph = nx.DiGraph() - self._debug = kwargs.get("debug", False) + self.graph = DiGraph() - # this holds the timing information for eache layer + # this holds the timing information for each layer self.times = {} - # a compiled list of steps to evaluate layers *in order* and free mem. - self.steps = [] + #: Speed up :meth:`compile()` call and avoid a multithreading issue(?) + #: that is occuring when accessing the dag in networkx. + self._cached_plans = {} + + #: the execution_plan of the last call to :meth:`compute()` + #: (not ``compile()``!), for debugging purposes. + self.last_plan = None - # This holds a cache of results for the _find_necessary_steps - # function, this helps speed up the compute call as well avoid - # a multithreading issue that is occuring when accessing the - # graph in networkx - self._necessary_steps_cache = {} + def _build_pydot(self, **kws): + from .plot import build_pydot + kws.setdefault("graph", self.graph) + + return build_pydot(**kws) def add_op(self, operation): """ Adds the given operation and its data requirements to the network graph - based on the name of the operation, the names of the operation's needs, and - the names of the data it provides. + based on the name of the operation, the names of the operation's needs, + and the names of the data it provides. :param Operation operation: Operation object to add. """ @@ -71,184 +179,444 @@ def add_op(self, operation): assert operation.provides is not None, "Operation's 'provides' must be named" # assert layer is only added once to graph - assert operation not in self.graph.nodes(), "Operation may only be added once" + assert operation not in self.graph.nodes, "Operation may only be added once" + + self._cached_plans = {} # add nodes and edges to graph describing the data needs for this layer for n in operation.needs: - self.graph.add_edge(DataPlaceholderNode(n), operation) + kw = {} + if isinstance(n, optional): + kw["optional"] = True + if isinstance(n, sideffect): + kw["sideffect"] = True + self.graph.add_edge(DataPlaceholderNode(n), operation, **kw) # add nodes and edges to graph describing what this layer provides for p in operation.provides: - self.graph.add_edge(operation, DataPlaceholderNode(p)) + kw = {} + if isinstance(n, sideffect): + kw["sideffect"] = True + self.graph.add_edge(operation, DataPlaceholderNode(p), **kw) - # clear compiled steps (must recompile after adding new layers) - self.steps = [] - - - def list_layers(self): - assert self.steps, "network must be compiled before listing layers." - return [(s.name, s) for s in self.steps if isinstance(s, Operation)] + def _build_execution_steps(self, dag, inputs, outputs): + """ + Create the list of operation-nodes & *instructions* evaluating all + operations & instructions needed a) to free memory and b) avoid + overwritting given intermediate inputs. - def show_layers(self): - """Shows info (name, needs, and provides) about all layers in this network.""" - for name, step in self.list_layers(): - print("layer_name: ", name) - print("\t", "needs: ", step.needs) - print("\t", "provides: ", step.provides) - print("") + :param dag: + The original dag, pruned; not broken. + :param outputs: + outp-names to decide whether to add (and which) del-instructions + In the list :class:`DeleteInstructions` steps (DA) are inserted between + operation nodes to reduce the memory footprint of solution. + A DA is inserted whenever a *need* is not used by any other *operation* + further down the DAG. + Note that since the `solutions` are not shared across `compute()` calls, + any memory-reductions are for as long as a single computation runs. - def compile(self): - """Create a set of steps for evaluating layers - and freeing memory as necessary""" + """ - # clear compiled steps - self.steps = [] + steps = [] # create an execution order such that each layer's needs are provided. - ordered_nodes = list(nx.dag.topological_sort(self.graph)) + ordered_nodes = iset(nx.topological_sort(dag)) - # add Operations evaluation steps, and instructions to free data. + # Add Operations evaluation steps, and instructions to free and "pin" + # data. for i, node in enumerate(ordered_nodes): if isinstance(node, DataPlaceholderNode): - continue + if node in inputs and dag.pred[node]: + # Command pinning only when there is another operation + # generating this data as output. + steps.append(PinInstruction(node)) elif isinstance(node, Operation): + steps.append(node) - # add layer to list of steps - self.steps.append(node) + # Keep all values in solution if not specific outputs asked. + if not outputs: + continue # Add instructions to delete predecessors as possible. A # predecessor may be deleted if it is a data placeholder that # is no longer needed by future Operations. - for predecessor in self.graph.predecessors(node): - if self._debug: - print("checking if node %s can be deleted" % predecessor) - predecessor_still_needed = False - for future_node in ordered_nodes[i+1:]: - if isinstance(future_node, Operation): - if predecessor in future_node.needs: - predecessor_still_needed = True - break - if not predecessor_still_needed: - if self._debug: - print(" adding delete instruction for %s" % predecessor) - self.steps.append(DeleteInstruction(predecessor)) + for need in self.graph.pred[node]: + log.debug("checking if node %s can be deleted", need) + for future_node in ordered_nodes[i + 1 :]: + if ( + isinstance(future_node, Operation) + and need in future_node.needs + ): + break + else: + if need not in outputs: + log.debug(" adding delete instruction for %s", need) + steps.append(DeleteInstruction(need)) + + else: + raise AssertionError("Unrecognized network graph node %r" % node) + + return steps + + def _collect_unsatisfied_operations(self, dag, inputs): + """ + Traverse topologically sorted dag to collect un-satisfied operations. + Unsatisfied operations are those suffering from ANY of the following: + + - They are missing at least one compulsory need-input. + Since the dag is ordered, as soon as we're on an operation, + all its needs have been accounted, so we can get its satisfaction. + + - Their provided outputs are not linked to any data in the dag. + An operation might not have any output link when :meth:`_prune_graph()` + has broken them, due to given intermediate inputs. + + :param dag: + a graph with broken edges those arriving to existing inputs + :param inputs: + an iterable of the names of the input values + return: + a list of unsatisfied operations to prune + """ + # To collect data that will be produced. + ok_data = set(inputs) + # To colect the map of operations --> satisfied-needs. + op_satisfaction = defaultdict(set) + # To collect the operations to drop. + unsatisfied = [] + for node in nx.topological_sort(dag): + if isinstance(node, Operation): + if not dag.adj[node]: + # Prune operations that ended up providing no output. + unsatisfied.append(node) + else: + # It's ok not to dig into edge-data("optional") here, + # we care about all needs, including broken ones. + real_needs = set( + n for n in node.needs if not isinstance(n, optional) + ) + if real_needs.issubset(op_satisfaction[node]): + # We have a satisfied operation; mark its output-data + # as ok. + ok_data.update(dag.adj[node]) + else: + # Prune operations with partial inputs. + unsatisfied.append(node) + elif isinstance(node, (DataPlaceholderNode, str)): # `str` are givens + if node in ok_data: + # mark satisfied-needs on all future operations + for future_op in dag.adj[node]: + op_satisfaction[future_op].add(node) else: - raise TypeError("Unrecognized network graph node") + raise AssertionError("Unrecognized network graph node %r" % node) + return unsatisfied - def _find_necessary_steps(self, outputs, inputs): + def _prune_graph(self, outputs, inputs): """ - Determines what graph steps need to pe run to get to the requested - outputs from the provided inputs. Eliminates steps that come before - (in topological order) any inputs that have been provided. Also - eliminates steps that are not on a path from the provided inputs to - the requested outputs. + Determines what graph steps need to run to get to the requested + outputs from the provided inputs. : + - Eliminate steps that are not on a path arriving to requested outputs. + - Eliminate unsatisfied operations: partial inputs or no outputs needed. - :param list outputs: + :param iterable outputs: A list of desired output names. This can also be ``None``, in which case the necessary steps are all graph nodes that are reachable from one of the provided inputs. - :param dict inputs: - A dictionary mapping names to values for all provided inputs. + :param iterable inputs: + The inputs names of all given inputs. - :returns: - Returns a list of all the steps that need to be run for the - provided inputs and requested outputs. + :return: + the *pruned_dag* + """ + dag = self.graph + + # Ignore input names that aren't in the graph. + graph_inputs = set(dag.nodes) & set(inputs) # unordered, iterated, but ok + + # Scream if some requested outputs aren't in the graph. + unknown_outputs = iset(outputs) - dag.nodes + if unknown_outputs: + raise ValueError( + "Unknown output node(s) requested: %s" % ", ".join(unknown_outputs) + ) + + broken_dag = dag.copy() # preserve net's graph + + # Break the incoming edges to all given inputs. + # + # Nodes producing any given intermediate inputs are unecessary + # (unless they are also used elsewhere). + # To discover which ones to prune, we break their incoming edges + # and they will drop out while collecting ancestors from the outputs. + broken_edges = set() # unordered, not iterated + for given in graph_inputs: + broken_edges.update(broken_dag.in_edges(given)) + broken_dag.remove_edges_from(broken_edges) + + # Drop stray input values and operations (if any). + broken_dag.remove_nodes_from(nx.isolates(broken_dag)) + + if outputs: + # If caller requested specific outputs, we can prune any + # unrelated nodes further up the dag. + ending_in_outputs = set() + for input_name in outputs: + ending_in_outputs.update(nx.ancestors(dag, input_name)) + broken_dag = broken_dag.subgraph(ending_in_outputs | set(outputs)) + + # Prune unsatisfied operations (those with partial inputs or no outputs). + unsatisfied = self._collect_unsatisfied_operations(broken_dag, inputs) + # Clone it so that it is picklable. + pruned_dag = dag.subgraph(broken_dag.nodes - unsatisfied).copy() + + return pruned_dag, broken_edges + + def compile(self, inputs=(), outputs=()): """ + Create or get from cache an execution-plan for the given inputs/outputs. - # return steps if it has already been computed before for this set of inputs and outputs - outputs = tuple(sorted(outputs)) if isinstance(outputs, (list, set)) else outputs - inputs_keys = tuple(sorted(inputs.keys())) - cache_key = (inputs_keys, outputs) - if cache_key in self._necessary_steps_cache: - return self._necessary_steps_cache[cache_key] + See :meth:`_prune_graph()` and :meth:`_build_execution_steps()` + for detailed description. - graph = self.graph - if not outputs: + :param inputs: + An iterable with the names of all the given inputs. - # If caller requested all outputs, the necessary nodes are all - # nodes that are reachable from one of the inputs. Ignore input - # names that aren't in the graph. - necessary_nodes = set() - for input_name in iter(inputs): - if graph.has_node(input_name): - necessary_nodes |= nx.descendants(graph, input_name) + :param outputs: + (optional) An iterable or the name of the output name(s). + If missing, requested outputs assumed all graph reachable nodes + from one of the given inputs. + :return: + the cached or fresh new execution-plan + """ + # outputs must be iterable + if not outputs: + outputs = () + elif isinstance(outputs, str): + outputs = (outputs,) + + # Make a stable cache-key + cache_key = (tuple(sorted(inputs)), tuple(sorted(outputs))) + if cache_key in self._cached_plans: + # An execution plan has been compiled before + # for the same inputs & outputs. + plan = self._cached_plans[cache_key] else: + # Build a new execution plan for the given inputs & outputs. + # + pruned_dag, broken_edges = self._prune_graph(outputs, inputs) + steps = self._build_execution_steps(pruned_dag, inputs, outputs) + plan = ExecutionPlan( + self, + tuple(inputs), + outputs, + pruned_dag, + tuple(broken_edges), + tuple(steps), + executed=iset(), + ) + + # Cache compilation results to speed up future runs + # with different values (but same number of inputs/outputs). + self._cached_plans[cache_key] = plan + + return plan + + def compute(self, named_inputs, outputs, method=None, overwrites_collector=None): + """ + Solve & execute the graph, sequentially or parallel. - # If the caller requested a subset of outputs, find any nodes that - # are made unecessary because we were provided with an input that's - # deeper into the network graph. Ignore input names that aren't - # in the graph. - unnecessary_nodes = set() - for input_name in iter(inputs): - if graph.has_node(input_name): - unnecessary_nodes |= nx.ancestors(graph, input_name) + :param dict named_inputs: + A dict of key/value pairs where the keys represent the data nodes + you want to populate, and the values are the concrete values you + want to set for the data node. - # Find the nodes we need to be able to compute the requested - # outputs. Raise an exception if a requested output doesn't - # exist in the graph. - necessary_nodes = set() - for output_name in outputs: - if not graph.has_node(output_name): - raise ValueError("graphkit graph does not have an output " - "node named %s" % output_name) - necessary_nodes |= nx.ancestors(graph, output_name) + :param list output: + once all necessary computations are complete. + If you set this variable to ``None``, all data nodes will be kept + and returned at runtime. - # Get rid of the unnecessary nodes from the set of necessary ones. - necessary_nodes -= unnecessary_nodes + :param method: + if ``"parallel"``, launches multi-threading. + Set when invoking a composed graph or by + :meth:`~NetworkOperation.set_execution_method()`. + :param overwrites_collector: + (optional) a mutable dict to be fillwed with named values. + If missing, values are simply discarded. - necessary_steps = [step for step in self.steps if step in necessary_nodes] + :returns: a dictionary of output data objects, keyed by name. + """ + try: + assert ( + isinstance(outputs, (list, tuple)) or outputs is None + ), "The outputs argument must be a list" + + # Build the execution plan. + self.last_plan = plan = self.compile(named_inputs.keys(), outputs) + + # start with fresh data solution. + solution = dict(named_inputs) + + plan.execute(solution, overwrites_collector, method) + + if outputs: + # Filter outputs to just return what's requested. + # Otherwise, eturn the whole solution as output, + # including input and intermediate data nodes. + # TODO: assert no other outputs exists due to DelInstructs. + solution = dict(i for i in solution.items() if i[0] in outputs) + + return solution + except Exception as ex: + ## Annotate exception with debugging aid on errorrs. + # + locs = locals() + err_aid = getattr(ex, "graphkit_aid", {}) + err_aid.setdefault("network", locs.get("self")) + err_aid.setdefault("plan", locs.get("plan")) + err_aid.setdefault("solution", locs.get("solution")) + setattr(ex, "graphkit_aid", err_aid) + raise + + +class ExecutionPlan( + namedtuple("_ExePlan", "net inputs outputs dag broken_edges steps executed"), + plot.Plotter, +): + """ + The result of the network's compilation phase. - # save this result in a precomputed cache for future lookup - self._necessary_steps_cache[cache_key] = necessary_steps + Note the execution plan's attributes are on purpose immutable tuples. - # Return an ordered list of the needed steps. - return necessary_steps + :ivar net: + The parent :class:`Network` + :ivar inputs: + A tuple with the names of the given inputs used to construct the plan. - def compute(self, outputs, named_inputs, method=None): - """ - Run the graph. Any inputs to the network must be passed in by name. + :ivar outputs: + A (possibly empy) tuple with the names of the requested outputs + used to construct the plan. - :param list output: The names of the data node you'd like to have returned - once all necessary computations are complete. - If you set this variable to ``None``, all - data nodes will be kept and returned at runtime. + :ivar dag: + The regular (not broken) *pruned* subgraph of net-graph. - :param dict named_inputs: A dict of key/value pairs where the keys - represent the data nodes you want to populate, - and the values are the concrete values you - want to set for the data node. + :ivar broken_edges: + Tuple of broken incoming edges to given data. + :ivar steps: + The tuple of operation-nodes & *instructions* needed to evaluate + the given inputs & asked outputs, free memory and avoid overwritting + any given intermediate inputs. + :ivar executed: + An empty set to collect all operations that have been executed so far. + """ - :returns: a dictionary of output data objects, keyed by name. - """ + @property + def broken_dag(self): + return nx.restricted_view(self.dag, nodes=(), edges=self.broken_edges) + + def _build_pydot(self, **kws): + from .plot import build_pydot + + clusters = None + if self.dag.nodes != self.net.graph.nodes: + clusters = {n: "after prunning" for n in self.dag.nodes} + mykws = { + "graph": self.net.graph, + "steps": self.steps, + "inputs": self.inputs, + "outputs": self.outputs, + "executed": self.executed, + "edge_props": { + e: {"color": "wheat", "penwidth": 2} for e in self.broken_edges + }, + "clusters": clusters, + } + mykws.update(kws) + + return build_pydot(**mykws) - # assert that network has been compiled - assert self.steps, "network must be compiled before calling compute." - assert isinstance(outputs, (list, tuple)) or outputs is None,\ - "The outputs argument must be a list" + def __repr__(self): + steps = ["\n +--%s" % s for s in self.steps] + return "ExecutionPlan(inputs=%s, outputs=%s, steps:%s)" % ( + self.inputs, + self.outputs, + "".join(steps), + ) + + def get_data_node(self, name): + """ + Retuen the data node from a graph using its name, or None. + """ + node = self.dag.nodes[name] + if isinstance(node, DataPlaceholderNode): + return node + def _can_schedule_operation(self, op): + """ + Determines if a Operation is ready to be scheduled for execution - # choose a method of execution - if method == "parallel": - return self._compute_thread_pool_barrier_method(named_inputs, - outputs) - else: - return self._compute_sequential_method(named_inputs, - outputs) + based on what has already been executed. + :param op: + The Operation object to check + :return: + A boolean indicating whether the operation may be scheduled for + execution based on what has already been executed. + """ + # Use `broken_dag` to allow executing operations after given inputs + # regardless of whether their producers have yet to run. + dependencies = set( + n for n in nx.ancestors(self.broken_dag, op) if isinstance(n, Operation) + ) + return dependencies.issubset(self.executed) + + def _can_evict_value(self, name): + """ + Determines if a DataPlaceholderNode is ready to be deleted from solution. - def _compute_thread_pool_barrier_method(self, named_inputs, outputs, - thread_pool_size=10): + :param name: + The name of the data node to check + :return: + A boolean indicating whether the data node can be deleted or not. + """ + data_node = self.get_data_node(name) + # Use `broken_dag` not to block a successor waiting for this data, + # since in any case will use a given input, not some pipe of this data. + return data_node and set(self.broken_dag.successors(data_node)).issubset( + self.executed + ) + + def _pin_data_in_solution(self, value_name, solution, inputs, overwrites): + value_name = str(value_name) + if overwrites is not None: + overwrites[value_name] = solution[value_name] + solution[value_name] = inputs[value_name] + + def _call_operation(self, op, solution): + try: + return op._compute(solution) + except Exception as ex: + ## Annotate exception with debugging aid on errors. + # + err_aid = getattr(ex, "graphkit_aid", {}) + err_aid.setdefault("plan", self) + setattr(ex, "graphkit_aid", err_aid) + raise + + def _execute_thread_pool_barrier_method( + self, inputs, solution, overwrites, thread_pool_size=10 + ): """ This method runs the graph using a parallel pool of thread executors. You may achieve lower total latency if your graph is sufficiently @@ -257,240 +625,118 @@ def _compute_thread_pool_barrier_method(self, named_inputs, outputs, from multiprocessing.dummy import Pool # if we have not already created a thread_pool, create one - if not hasattr(self, "_thread_pool"): - self._thread_pool = Pool(thread_pool_size) - pool = self._thread_pool - - cache = {} - cache.update(named_inputs) - necessary_nodes = self._find_necessary_steps(outputs, named_inputs) - - # this keeps track of all nodes that have already executed - has_executed = set() + if not hasattr(self.net, "_thread_pool"): + self.net._thread_pool = Pool(thread_pool_size) + pool = self.net._thread_pool # with each loop iteration, we determine a set of operations that can be # scheduled, then schedule them onto a thread pool, then collect their - # results onto a memory cache for use upon the next iteration. + # results onto a memory solution for use upon the next iteration. while True: # the upnext list contains a list of operations for scheduling # in the current round of scheduling upnext = [] - for node in necessary_nodes: - # only delete if all successors for the data node have been executed - if isinstance(node, DeleteInstruction): - if ready_to_delete_data_node(node, - has_executed, - self.graph): - if node in cache: - cache.pop(node) - - # continue if this node is anything but an operation node - if not isinstance(node, Operation): - continue - - if ready_to_schedule_operation(node, has_executed, self.graph) \ - and node not in has_executed: + for node in self.steps: + if ( + isinstance(node, Operation) + and self._can_schedule_operation(node) + and node not in self.executed + ): upnext.append(node) - + elif isinstance(node, DeleteInstruction): + # Only delete if all successors for the data node + # have been executed. + # An optional need may not have a value in the solution. + if node in solution and self._can_evict_value(node): + log.debug("removing data '%s' from solution.", node) + del solution[node] + elif isinstance(node, PinInstruction): + # Always and repeatedely pin the value, even if not all + # providers of the data have executed. + # An optional need may not have a value in the solution. + if node in solution: + self._pin_data_in_solution(node, solution, inputs, overwrites) # stop if no nodes left to schedule, exit out of the loop if len(upnext) == 0: break + ## TODO: accept pool from caller done_iterator = pool.imap_unordered( - lambda op: (op,op._compute(cache)), - upnext) - for op, result in done_iterator: - cache.update(result) - has_executed.add(op) + (lambda op: (op, self._call_operation(op, solution))), upnext + ) - if not outputs: - return cache - else: - return {k: cache[k] for k in iter(cache) if k in outputs} + for op, result in done_iterator: + solution.update(result) + self.executed.add(op) - def _compute_sequential_method(self, named_inputs, outputs): + def _execute_sequential_method(self, inputs, solution, overwrites): """ This method runs the graph one operation at a time in a single thread """ - # start with fresh data cache - cache = {} - - # add inputs to data cache - cache.update(named_inputs) - - # Find the subset of steps we need to run to get to the requested - # outputs from the provided inputs. - all_steps = self._find_necessary_steps(outputs, named_inputs) - self.times = {} - for step in all_steps: + for step in self.steps: if isinstance(step, Operation): - if self._debug: - print("-"*32) - print("executing step: %s" % step.name) + log.debug("%sexecuting step: %s", "-" * 32, step.name) # time execution... t0 = time.time() # compute layer outputs - layer_outputs = step._compute(cache) + layer_outputs = self._call_operation(step, solution) - # add outputs to cache - cache.update(layer_outputs) + # add outputs to solution + solution.update(layer_outputs) + self.executed.add(step) # record execution time t_complete = round(time.time() - t0, 5) self.times[step.name] = t_complete - if self._debug: - print("step completion time: %s" % t_complete) + log.debug("step completion time: %s", t_complete) - # Process DeleteInstructions by deleting the corresponding data - # if possible. elif isinstance(step, DeleteInstruction): + # Cache value may be missing if it is optional. + if step in solution: + log.debug("removing data '%s' from solution.", step) + del solution[step] - if outputs and step not in outputs: - # Some DeleteInstruction steps may not exist in the cache - # if they come from optional() needs that are not privoded - # as inputs. Make sure the step exists before deleting. - if step in cache: - if self._debug: - print("removing data '%s' from cache." % step) - cache.pop(step) - + elif isinstance(step, PinInstruction): + self._pin_data_in_solution(step, solution, inputs, overwrites) else: - raise TypeError("Unrecognized instruction.") + raise AssertionError("Unrecognized instruction.%r" % step) - if not outputs: - # Return the whole cache as output, including input and - # intermediate data nodes. - return cache - - else: - # Filter outputs to just return what's needed. - # Note: list comprehensions exist in python 2.7+ - return {k: cache[k] for k in iter(cache) if k in outputs} - - - def plot(self, filename=None, show=False): + def execute(self, solution, overwrites=None, method=None): """ - Plot the graph. - - params: - :param str filename: - Write the output to a png, pdf, or graphviz dot file. The extension - controls the output format. - - :param boolean show: - If this is set to True, use matplotlib to show the graph diagram - (Default: False) - - :returns: - An instance of the pydot graph - + :param solution: + a mutable maping to collect the results and that must contain also + the given input values for at least the compulsory inputs that + were specified when the plan was built (but cannot enforce that!). + + :param overwrites: + (optional) a mutable dict to collect calculated-but-discarded values + because they were "pinned" by input vaules. + If missing, the overwrites values are simply discarded. """ - import pydot - import matplotlib.pyplot as plt - import matplotlib.image as mpimg - - assert self.graph is not None - - def get_node_name(a): - if isinstance(a, DataPlaceholderNode): - return a - return a.name - - g = pydot.Dot(graph_type="digraph") - - # draw nodes - for nx_node in self.graph.nodes(): - if isinstance(nx_node, DataPlaceholderNode): - node = pydot.Node(name=nx_node, shape="rect") - else: - node = pydot.Node(name=nx_node.name, shape="circle") - g.add_node(node) - - # draw edges - for src, dst in self.graph.edges(): - src_name = get_node_name(src) - dst_name = get_node_name(dst) - edge = pydot.Edge(src=src_name, dst=dst_name) - g.add_edge(edge) - - # save plot - if filename: - basename, ext = os.path.splitext(filename) - with open(filename, "w") as fh: - if ext.lower() == ".png": - fh.write(g.create_png()) - elif ext.lower() == ".dot": - fh.write(g.to_string()) - elif ext.lower() in [".jpg", ".jpeg"]: - fh.write(g.create_jpeg()) - elif ext.lower() == ".pdf": - fh.write(g.create_pdf()) - elif ext.lower() == ".svg": - fh.write(g.create_svg()) - else: - raise Exception("Unknown file format for saving graph: %s" % ext) - - # display graph via matplotlib - if show: - png = g.create_png() - sio = StringIO(png) - img = mpimg.imread(sio) - plt.imshow(img, aspect="equal") - plt.show() - - return g - + # Clean executed operation from any previous execution. + self.executed.clear() -def ready_to_schedule_operation(op, has_executed, graph): - """ - Determines if a Operation is ready to be scheduled for execution based on - what has already been executed. + # choose a method of execution + executor = ( + self._execute_thread_pool_barrier_method + if method == "parallel" + else self._execute_sequential_method + ) - Args: - op: - The Operation object to check - has_executed: set - A set containing all operations that have been executed so far - graph: - The networkx graph containing the operations and data nodes - Returns: - A boolean indicating whether the operation may be scheduled for - execution based on what has already been executed. - """ - dependencies = set(filter(lambda v: isinstance(v, Operation), - nx.ancestors(graph, op))) - return dependencies.issubset(has_executed) + # clone and keep orignal inputs in solution intact + executor(dict(solution), solution, overwrites) -def ready_to_delete_data_node(name, has_executed, graph): - """ - Determines if a DataPlaceholderNode is ready to be deleted from the - cache. + # return it, but caller can also see the results in `solution` dict. + return solution - Args: - name: - The name of the data node to check - has_executed: set - A set containing all operations that have been executed so far - graph: - The networkx graph containing the operations and data nodes - Returns: - A boolean indicating whether the data node can be deleted or not. - """ - data_node = get_data_node(name, graph) - return set(graph.successors(data_node)).issubset(has_executed) -def get_data_node(name, graph): - """ - Gets a data node from a graph using its name - """ - for node in graph.nodes(): - if node == name and isinstance(node, DataPlaceholderNode): - return node - return None +# TODO: maybe class Solution(object): +# values = {} +# overwrites = None diff --git a/graphkit/plot.py b/graphkit/plot.py new file mode 100644 index 00000000..1b5daa0e --- /dev/null +++ b/graphkit/plot.py @@ -0,0 +1,427 @@ +# Copyright 2016, Yahoo Inc. +# Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. +""" Plotting graphkit graps""" +import io +import logging +import os + + +log = logging.getLogger(__name__) + + +class Plotter(object): + """ + Classes wishing to plot their graphs should inherit this and ... + + implement property ``_plot`` to return a "partial" callable that somehow + ends up calling :func:`plot.render_pydot()` with the `graph` or any other + args binded appropriately. + The purpose is to avoid copying this function & documentation here around. + """ + + def plot(self, filename=None, show=False, **kws): + """ + :param str filename: + Write diagram into a file. + Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` + call :func:`plot.supported_plot_formats()` for more. + :param show: + If it evaluates to true, opens the diagram in a matplotlib window. + If it equals `-1`, it plots but does not open the Window. + :param inputs: + an optional name list, any nodes in there are plotted + as a "house" + :param outputs: + an optional name list, any nodes in there are plotted + as an "inverted-house" + :param solution: + an optional dict with values to annotate nodes, drawn "filled" + (currently content not shown, but node drawn as "filled") + :param executed: + an optional container with operations executed, drawn "filled" + :param title: + an optional string to display at the bottom of the graph + :param node_props: + an optional nested dict of Grapvhiz attributes for certain nodes + :param edge_props: + an optional nested dict of Grapvhiz attributes for certain edges + :param clusters: + an optional mapping of nodes --> cluster-names, to group them + + :return: + A ``pydot.Dot`` instance. + NOTE that the returned instance is monkeypatched to support + direct rendering in *jupyter cells* as SVG. + + + Note that the `graph` argument is absent - Each Plotter provides + its own graph internally; use directly :func:`render_pydot()` to provide + a different graph. + + .. image:: images/GraphkitLegend.svg + :alt: Graphkit Legend + + *NODES:* + + oval + function + egg + subgraph operation + house + given input + inversed-house + asked output + polygon + given both as input & asked as output (what?) + square + intermediate data, neither given nor asked. + red frame + delete-instruction, to free up memory. + blue frame + pinned-instruction, not to overwrite intermediate inputs. + filled + data node has a value in `solution` OR function has been executed. + thick frame + function/data node in execution `steps`. + + *ARROWS* + + solid black arrows + dependencies (source-data *need*-ed by target-operations, + sources-operations *provides* target-data) + dashed black arrows + optional needs + blue arrows + sideffect needs/provides + wheat arrows + broken dependency (``provide``) during pruning + green-dotted arrows + execution steps labeled in succession + + + To generate the **legend**, see :func:`legend()`. + + **Sample code:** + + >>> from graphkit import compose, operation + >>> from graphkit.modifiers import optional + >>> from operator import add + + >>> graphop = compose(name="graphop")( + ... operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), + ... operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])(lambda a, b=1: a-b), + ... operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), + ... ) + + >>> graphop.plot(show=True); # plot just the graph in a matplotlib window # doctest: +SKIP + >>> inputs = {'a': 1, 'b1': 2} + >>> solution = graphop(inputs) # now plots will include the execution-plan + + >>> graphop.plot('plot1.svg', inputs=inputs, outputs=['asked', 'b1'], solution=solution); # doctest: +SKIP + >>> dot = graphop.plot(solution=solution); # just get the `pydoit.Dot` object, renderable in Jupyter + >>> print(dot) + digraph G { + fontname=italic; + label=graphop; + a [fillcolor=wheat, shape=invhouse, style=filled]; + ... + ... + """ + dot = self._build_pydot(**kws) + return render_pydot(dot, filename=filename, show=show) + + def _build_pydot(self, **kws): + raise AssertionError("Must implement that!") + + +def _is_class_value_in_list(lst, cls, value): + return any(isinstance(i, cls) and i == value for i in lst) + + +def _merge_conditions(*conds): + """combines conditions as a choice in binary range, eg, 2 conds --> [0, 3]""" + return sum(int(bool(c)) << i for i, c in enumerate(conds)) + + +def _apply_user_props(dotobj, user_props, key): + if user_props and key in user_props: + dotobj.get_attributes().update(user_props[key]) + # Delete it, to report unmatched ones, AND not to annotate `steps`. + del user_props[key] + + +def _report_unmatched_user_props(user_props, kind): + if user_props and log.isEnabledFor(logging.WARNING): + unmatched = "\n ".join(str(i) for i in user_props.items()) + log.warning("Unmatched `%s_props`:\n +--%s", kind, unmatched) + + +def _monkey_patch_for_jupyter(pydot): + # Ensure Dot nstance render in Jupyter + # (see pydot/pydot#220) + if not hasattr(pydot.Dot, "_repr_svg_"): + + def make_svg(self): + return self.create_svg().decode() + + # monkey patch class + pydot.Dot._repr_svg_ = make_svg + + +def build_pydot( + graph, + steps=None, + inputs=None, + outputs=None, + solution=None, + executed=None, + title=None, + node_props=None, + edge_props=None, + clusters=None, +): + """ + Build a *Graphviz* out of a Network graph/steps/inputs/outputs and return it. + + See :meth:`Plotter.plot()` for the arguments, sample code, and + the legend of the plots. + """ + import pydot + from .base import NetworkOperation, Operation + from .modifiers import optional + from .network import DeleteInstruction, PinInstruction + + _monkey_patch_for_jupyter(pydot) + + assert graph is not None + + steps_thickness = 3 + fill_color = "wheat" + steps_color = "#009999" + new_clusters = {} + + def append_or_cluster_node(dot, nx_node, node): + if not clusters or not nx_node in clusters: + dot.add_node(node) + else: + cluster_name = clusters[nx_node] + node_cluster = new_clusters.get(cluster_name) + if not node_cluster: + node_cluster = new_clusters[cluster_name] = pydot.Cluster( + cluster_name, label=cluster_name + ) + node_cluster.add_node(node) + + def append_any_clusters(dot): + for cluster in new_clusters.values(): + dot.add_subgraph(cluster) + + def get_node_name(a): + if isinstance(a, Operation): + return a.name + return a + + dot = pydot.Dot(graph_type="digraph", label=title, fontname="italic") + + # draw nodes + for nx_node in graph.nodes: + if isinstance(nx_node, str): + kw = {} + + # FrameColor change by step type + if steps and nx_node in steps: + choice = _merge_conditions( + _is_class_value_in_list(steps, DeleteInstruction, nx_node), + _is_class_value_in_list(steps, PinInstruction, nx_node), + ) + # 0 is singled out because `nx_node` exists in `steps`. + color = "NOPE #990000 blue purple".split()[choice] + kw = {"color": color, "penwidth": steps_thickness} + + # SHAPE change if with inputs/outputs. + # tip: https://graphviz.gitlab.io/_pages/doc/info/shapes.html + choice = _merge_conditions( + inputs and nx_node in inputs, outputs and nx_node in outputs + ) + shape = "rect invhouse house hexagon".split()[choice] + + # LABEL change with solution. + if solution and nx_node in solution: + kw["style"] = "filled" + kw["fillcolor"] = fill_color + # kw["tooltip"] = str(solution.get(nx_node)) # not working :-() + node = pydot.Node(name=nx_node, shape=shape, **kw) + else: # Operation + kw = {"fontname": "italic"} + + if steps and nx_node in steps: + kw["penwdth"] = steps_thickness + shape = "egg" if isinstance(nx_node, NetworkOperation) else "oval" + if executed and nx_node in executed: + kw["style"] = "filled" + kw["fillcolor"] = fill_color + node = pydot.Node(name=nx_node.name, shape=shape, **kw) + + _apply_user_props(node, node_props, key=node.get_name()) + + append_or_cluster_node(dot, nx_node, node) + + _report_unmatched_user_props(node_props, "node") + + append_any_clusters(dot) + + # draw edges + for src, dst, data in graph.edges(data=True): + src_name = get_node_name(src) + dst_name = get_node_name(dst) + + kw = {} + if data.get("optional"): + kw["style"] = "dashed" + if data.get("sideffect"): + kw["color"] = "blue" + + # `splines=ortho` not working :-() + edge = pydot.Edge(src=src_name, dst=dst_name, splines="ortho", **kw) + + _apply_user_props(edge, edge_props, key=(src, dst)) + + dot.add_edge(edge) + + _report_unmatched_user_props(edge_props, "edge") + + # draw steps sequence + if steps and len(steps) > 1: + it1 = iter(steps) + it2 = iter(steps) + next(it2) + for i, (src, dst) in enumerate(zip(it1, it2), 1): + src_name = get_node_name(src) + dst_name = get_node_name(dst) + edge = pydot.Edge( + src=src_name, + dst=dst_name, + label=str(i), + style="dotted", + color=steps_color, + fontcolor=steps_color, + fontname="bold", + fontsize=18, + penwidth=steps_thickness, + arrowhead="vee", + splines=True, + ) + dot.add_edge(edge) + + return dot + + +def supported_plot_formats(): + """return automatically all `pydot` extensions""" + import pydot + + return [".%s" % f for f in pydot.Dot().formats] + + +def render_pydot(dot, filename=None, show=False): + """ + Plot a *Graphviz* dot in a matplotlib, in file or return it for Jupyter. + + :param dot: + the pre-built *Graphviz* dot instance + :param str filename: + Write diagram into a file. + Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` + call :func:`plot.supported_plot_formats()` for more. + :param show: + If it evaluates to true, opens the diagram in a matplotlib window. + If it equals `-1`, it returns the image but does not open the Window. + + :return: + the matplotlib image if ``show=-1``, or the `dot`. + + See :meth:`Plotter.plot()` for sample code. + """ + # TODO: research https://plot.ly/~empet/14007.embed + # Save plot + # + if filename: + formats = supported_plot_formats() + _basename, ext = os.path.splitext(filename) + if not ext.lower() in formats: + raise ValueError( + "Unknown file format for saving graph: %s" + " File extensions must be one of: %s" % (ext, " ".join(formats)) + ) + + dot.write(filename, format=ext.lower()[1:]) + + ## Display graph via matplotlib + # + if show: + import matplotlib.pyplot as plt + import matplotlib.image as mpimg + + png = dot.create_png() + sio = io.BytesIO(png) + img = mpimg.imread(sio) + if show != -1: + plt.imshow(img, aspect="equal") + plt.show() + + return img + + return dot + + +def legend(filename=None, show=None): + """Generate a legend for all plots (see Plotter.plot() for args)""" + import pydot + + _monkey_patch_for_jupyter(pydot) + + ## From https://stackoverflow.com/questions/3499056/making-a-legend-key-in-graphviz + dot_text = """ + digraph { + rankdir=LR; + subgraph cluster_legend { + label="Graphkit Legend"; + + operation [shape=oval fontname=italic]; + graphop [shape=egg label="graph operation" fontname=italic]; + insteps [penwidth=3 label="execution step" fontname=italic]; + executed [style=filled fillcolor=wheat fontname=italic]; + operation -> graphop -> insteps -> executed [style=invis]; + + data [shape=rect]; + input [shape=invhouse]; + output [shape=house]; + inp_out [shape=hexagon label="inp+out"]; + evicted [shape=rect penwidth=3 color="#990000"]; + pinned [shape=rect penwidth=3 color="blue"]; + evpin [shape=rect penwidth=3 color=purple label="evict+pin"]; + sol [shape=rect style=filled fillcolor=wheat label="in solution"]; + data -> input -> output -> inp_out -> evicted -> pinned -> evpin -> sol [style=invis]; + + e1 [style=invis] e2 [color=invis label="dependency"]; + e1 -> e2; + e3 [color=invis label="optional"]; + e2 -> e3 [style=dashed]; + e33 [color=invis label="sideffect"]; + e3 -> e33 [color=blue]; + e4 [color=invis penwidth=3 label="pruned dependency"]; + e33 -> e4 [color=wheat penwidth=2]; + e5 [color=invis penwidth=4 label="execution sequence"]; + e4 -> e5 [color="#009999" penwidth=4 style=dotted arrowhead=vee label=1 fontcolor="#009999"]; + } + } + """ + + dot = pydot.graph_from_dot_data(dot_text)[0] + # clus = pydot.Cluster("Graphkit legend", label="Graphkit legend") + # dot.add_subgraph(clus) + + # nodes = dot.Node() + # clus.add_node("operation") + + return render_pydot(dot, filename=filename, show=show) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..aa85f913 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,24 @@ +## Python's setup.cfg for tool defaults: +# +[bdist_wheel] +universal = 1 + + +[tool:pytest] +# See http://doc.pytest.org/en/latest/mark.html#mark +markers = + slow: marks tests as slow, select them with `-m slow` or `-m 'not slow'` + +# TODO: enable doctests in README.md. +addopts = graphkit test/ docs/source/ README.md + # Faciltate developer, rum'em all with -m 'slow or not slow'. + -m 'not slow' + --doctest-report ndiff + --doctest-continue-on-failure + # --doctest-ignore-import-errors + --doctest-modules + --doctest-glob=*.md + --doctest-glob=*.rst + --cov-fail-under=80 +doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS + diff --git a/setup.py b/setup.py index bd7883f4..e9cb145c 100644 --- a/setup.py +++ b/setup.py @@ -1,56 +1,77 @@ #!/usr/bin/env python # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. +import io import os import re -import io from setuptools import setup -LONG_DESCRIPTION = """ -GraphKit is a lightweight Python module for creating and running ordered graphs -of computations, where the nodes of the graph correspond to computational -operations, and the edges correspond to output --> input dependencies between -those operations. Such graphs are useful in computer vision, machine learning, -and many other domains. -""" + +with open("README.md") as f: + long_description = f.read() # Grab the version using convention described by flask # https://github.com/pallets/flask/blob/master/setup.py#L10 -with io.open('graphkit/__init__.py', 'rt', encoding='utf8') as f: - version = re.search(r'__version__ = \'(.*?)\'', f.read()).group(1) +with io.open("graphkit/__init__.py", "rt", encoding="utf8") as f: + version = re.search(r'__version__ = "(.*?)"', f.read()).group(1) + +plot_reqs = ["matplotlib", "pydot"] # to test plot # to test plot +test_reqs = plot_reqs + ["pytest", "pytest-cov", "pytest-sphinx"] setup( - name='graphkit', - version=version, - description='Lightweight computation graphs for Python', - long_description=LONG_DESCRIPTION, - author='Huy Nguyen, Arel Cordero, Pierre Garrigues, Rob Hess, Tobi Baumgartner, Clayton Mellina', - author_email='huyng@yahoo-inc.com', - url='http://github.com/yahoo/graphkit', - packages=['graphkit'], - install_requires=['networkx'], - extras_require={ - 'plot': ['pydot', 'matplotlib'] - }, - tests_require=['numpy'], - license='Apache-2.0', - keywords=['graph', 'computation graph', 'DAG', 'directed acyclical graph'], - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: Apache Software License', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'Natural Language :: English', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Operating System :: POSIX', - 'Operating System :: Unix', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Topic :: Scientific/Engineering', - 'Topic :: Software Development' + name="graphkit", + version=version, + description="Lightweight computation graphs for Python", + long_description=long_description, + author="Huy Nguyen, Arel Cordero, Pierre Garrigues, Rob Hess, " + "Tobi Baumgartner, Clayton Mellina, ankostis@gmail.com", + author_email="huyng@yahoo-inc.com", + url="http://github.com/yahoo/graphkit", + project_urls={ + "Documentation": "https://pythonhosted.org/graphkit/", + "Release Notes": "https://pythonhosted.org/graphkit/changes.html", + "Sources": "https://github.com/yahoo/graphkit", + "Bug Tracker": "https://github.com/yahoo/graphkit/issues", + }, + packages=["graphkit"], + install_requires=[ + "networkx; python_version >= '3.5'", + "networkx == 2.2; python_version < '3.5'", + "boltons", # for IndexSet + ], + extras_require={"plot": plot_reqs, "test": test_reqs}, + tests_require=test_reqs, + license="Apache-2.0", + keywords=[ + "graph", + "computation graph", + "DAG", + "directed acyclical graph", + "executor", + "scheduler", + "etl", + "workflow", + "pipeline", + ], + classifiers=[ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Natural Language :: English", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Operating System :: POSIX", + "Operating System :: Unix", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Scientific/Engineering", + "Topic :: Software Development", ], - platforms='Windows,Linux,Solaris,Mac OS-X,Unix' + zip_safe=True, + platforms="Windows,Linux,Solaris,Mac OS-X,Unix", ) diff --git a/test/test_doc.py b/test/test_doc.py new file mode 100644 index 00000000..f54cf76f --- /dev/null +++ b/test/test_doc.py @@ -0,0 +1,24 @@ +# Copyright 2016, Yahoo Inc. +# Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. +import os.path as osp +import subprocess +import sys + + +def test_README_as_PyPi_landing_page(monkeypatch): + from docutils import core as dcore + + proj_path = osp.join(osp.dirname(__file__), "..") + long_desc = subprocess.check_output( + "python setup.py --long-description".split(), cwd=proj_path + ) + assert long_desc + + monkeypatch.setattr(sys, "exit", lambda *args: None) + dcore.publish_string( + long_desc, + enable_exit_status=False, + settings_overrides={ # see `docutils.frontend` for more. + "halt_level": 2 # 2=WARN, 1=INFO + }, + ) diff --git a/test/test_functional.py b/test/test_functional.py new file mode 100644 index 00000000..13cd1270 --- /dev/null +++ b/test/test_functional.py @@ -0,0 +1,34 @@ +# Copyright 2016, Yahoo Inc. +# Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. + +import pytest + +from graphkit import Operation, operation + + +@pytest.fixture(params=[None, ["some"]]) +def opname(request): + return request.param + + +@pytest.fixture(params=[None, ["some"]]) +def opneeds(request): + return request.param + + +@pytest.fixture(params=[None, ["some"]]) +def opprovides(request): + return request.param + + +def test_operation_repr(opname, opneeds, opprovides): + # Simply check __repr__() does not crash on partial attributes. + + kw = locals().copy() + kw = {name[2:]: arg for name, arg in kw.items()} + + op = operation(**kw) + str(op) + + op = Operation(**kw) + str(op) diff --git a/test/test_graphkit.py b/test/test_graphkit.py index bd97b317..2184ee9e 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -3,40 +3,70 @@ import math import pickle - +import sys +from operator import add, floordiv, mul, sub from pprint import pprint -from operator import add -from numpy.testing import assert_raises -import graphkit.network as network +import pytest + import graphkit.modifiers as modifiers -from graphkit import operation, compose, Operation +import graphkit.network as network +from graphkit import Operation, compose, operation +from graphkit.network import DeleteInstruction + + +def scream(*args, **kwargs): + raise AssertionError( + "Must not have run!\n args: %s\n kwargs: %s", (args, kwargs) + ) + -def test_network(): +def identity(x): + return x + + +def filtdict(d, *keys): + """ + Keep dict items with the given keys + + >>> filtdict({"a": 1, "b": 2}, "b") + {'b': 2} + """ + return type(d)(i for i in d.items() if i[0] in keys) + + +def test_network_smoke(): # Sum operation, late-bind compute function - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum_ab')(add) + sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum_ab")(add) # sum_op1 is callable - print(sum_op1(1, 2)) + assert sum_op1(1, 2) == 3 # Multiply operation, decorate in-place - @operation(name='mul_op1', needs=['sum_ab', 'b'], provides='sum_ab_times_b') + @operation(name="mul_op1", needs=["sum_ab", "b"], provides="sum_ab_times_b") def mul_op1(a, b): return a * b # mul_op1 is callable - print(mul_op1(1, 2)) + assert mul_op1(1, 2) == 2 # Pow operation - @operation(name='pow_op1', needs='sum_ab', provides=['sum_ab_p1', 'sum_ab_p2', 'sum_ab_p3'], params={'exponent': 3}) + @operation( + name="pow_op1", + needs="sum_ab", + provides=["sum_ab_p1", "sum_ab_p2", "sum_ab_p3"], + params={"exponent": 3}, + ) def pow_op1(a, exponent=2): - return [math.pow(a, y) for y in range(1, exponent+1)] + return [math.pow(a, y) for y in range(1, exponent + 1)] - print(pow_op1._compute({'sum_ab':2}, ['sum_ab_p2'])) + assert pow_op1._compute({"sum_ab": 2}, ["sum_ab_p2"]) == {"sum_ab_p2": 4.0} # Partial operation that is bound at a later time - partial_op = operation(name='sum_op2', needs=['sum_ab_p1', 'sum_ab_p2'], provides='p1_plus_p2') + partial_op = operation( + name="sum_op2", needs=["sum_ab_p1", "sum_ab_p2"], provides="p1_plus_p2" + ) # Bind the partial operation sum_op2 = partial_op(add) @@ -44,26 +74,39 @@ def pow_op1(a, exponent=2): # Sum operation, early-bind compute function sum_op_factory = operation(add) - sum_op3 = sum_op_factory(name='sum_op3', needs=['a', 'b'], provides='sum_ab2') + sum_op3 = sum_op_factory(name="sum_op3", needs=["a", "b"], provides="sum_ab2") # sum_op3 is callable - print(sum_op3(5, 6)) + assert sum_op3(5, 6) == 11 # compose network - net = compose(name='my network')(sum_op1, mul_op1, pow_op1, sum_op2, sum_op3) + net = compose(name="my network")(sum_op1, mul_op1, pow_op1, sum_op2, sum_op3) # # Running the network # # get all outputs - pprint(net({'a': 1, 'b': 2})) + exp = { + "a": 1, + "b": 2, + "p1_plus_p2": 12.0, + "sum_ab": 3, + "sum_ab2": 3, + "sum_ab_p1": 3.0, + "sum_ab_p2": 9.0, + "sum_ab_p3": 27.0, + "sum_ab_times_b": 6, + } + assert net({"a": 1, "b": 2}) == exp # get specific outputs - pprint(net({'a': 1, 'b': 2}, outputs=["sum_ab_times_b"])) + exp = {"sum_ab_times_b": 6} + assert net({"a": 1, "b": 2}, outputs=["sum_ab_times_b"]) == exp # start with inputs already computed - pprint(net({"sum_ab": 1, "b": 2}, outputs=["sum_ab_times_b"])) + exp = {"sum_ab_times_b": 2} + assert net({"sum_ab": 1, "b": 2}, outputs=["sum_ab_times_b"]) == exp # visualize network graph # net.plot(show=True) @@ -71,36 +114,83 @@ def pow_op1(a, exponent=2): def test_network_simple_merge(): - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['a', 'b'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) - net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) - pprint(net1({'a': 1, 'b': 2, 'c': 4})) - - sum_op4 = operation(name='sum_op1', needs=['d', 'e'], provides='a')(add) - sum_op5 = operation(name='sum_op2', needs=['a', 'f'], provides='b')(add) - net2 = compose(name='my network 2')(sum_op4, sum_op5) - pprint(net2({'d': 1, 'e': 2, 'f': 4})) - - net3 = compose(name='merged')(net1, net2) - pprint(net3({'c': 5, 'd': 1, 'e': 2, 'f': 4})) + sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add) + sum_op2 = operation(name="sum_op2", needs=["a", "b"], provides="sum2")(add) + sum_op3 = operation(name="sum_op3", needs=["sum1", "c"], provides="sum3")(add) + net1 = compose(name="my network 1")(sum_op1, sum_op2, sum_op3) + + exp = {"a": 1, "b": 2, "c": 4, "sum1": 3, "sum2": 3, "sum3": 7} + sol = net1({"a": 1, "b": 2, "c": 4}) + assert sol == exp + + sum_op4 = operation(name="sum_op1", needs=["d", "e"], provides="a")(add) + sum_op5 = operation(name="sum_op2", needs=["a", "f"], provides="b")(add) + + net2 = compose(name="my network 2")(sum_op4, sum_op5) + exp = {"a": 3, "b": 7, "d": 1, "e": 2, "f": 4} + sol = net2({"d": 1, "e": 2, "f": 4}) + assert sol == exp + + net3 = compose(name="merged")(net1, net2) + exp = { + "a": 3, + "b": 7, + "c": 5, + "d": 1, + "e": 2, + "f": 4, + "sum1": 10, + "sum2": 10, + "sum3": 15, + } + sol = net3({"c": 5, "d": 1, "e": 2, "f": 4}) + assert sol == exp def test_network_deep_merge(): - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['a', 'b'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) - net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) - pprint(net1({'a': 1, 'b': 2, 'c': 4})) - - sum_op4 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op5 = operation(name='sum_op4', needs=['sum1', 'b'], provides='sum2')(add) - net2 = compose(name='my network 2')(sum_op4, sum_op5) - pprint(net2({'a': 1, 'b': 2})) + sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add) + sum_op2 = operation(name="sum_op2", needs=["a", "b"], provides="sum2")(add) + sum_op3 = operation(name="sum_op3", needs=["sum1", "c"], provides="sum3")(add) + net1 = compose(name="my network 1")(sum_op1, sum_op2, sum_op3) + + exp = {"a": 1, "b": 2, "c": 4, "sum1": 3, "sum2": 3, "sum3": 7} + assert net1({"a": 1, "b": 2, "c": 4}) == exp + + sum_op4 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add) + sum_op5 = operation(name="sum_op4", needs=["sum1", "b"], provides="sum2")(add) + net2 = compose(name="my network 2")(sum_op4, sum_op5) + exp = {"a": 1, "b": 2, "sum1": 3, "sum2": 5} + assert net2({"a": 1, "b": 2}) == exp + + net3 = compose(name="merged", merge=True)(net1, net2) + exp = {"a": 1, "b": 2, "c": 4, "sum1": 3, "sum2": 3, "sum3": 7} + assert net3({"a": 1, "b": 2, "c": 4}) == exp + + +def test_network_merge_in_doctests(): + def abspow(a, p): + c = abs(a) ** p + return c + + graphop = compose(name="graphop")( + operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), + operation(name="sub1", needs=["a", "ab"], provides=["a_minus_ab"])(sub), + operation( + name="abspow1", + needs=["a_minus_ab"], + provides=["abs_a_minus_ab_cubed"], + params={"p": 3}, + )(abspow), + ) - net3 = compose(name='merged', merge=True)(net1, net2) - pprint(net3({'a': 1, 'b': 2, 'c': 4})) + another_graph = compose(name="another_graph")( + operation(name="mul1", needs=["a", "b"], provides=["ab"])(mul), + operation(name="mul2", needs=["c", "ab"], provides=["cab"])(mul), + ) + merged_graph = compose(name="merged_graph", merge=True)(graphop, another_graph) + assert merged_graph.needs + assert merged_graph.provides def test_input_based_pruning(): @@ -112,16 +202,16 @@ def test_input_based_pruning(): # Set up a net such that if sum1 and sum2 are provided directly, we don't # need to provide a and b. - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['a', 'b'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['sum1', 'sum2'], provides='sum3')(add) - net = compose(name='test_net')(sum_op1, sum_op2, sum_op3) + sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add) + sum_op2 = operation(name="sum_op2", needs=["a", "b"], provides="sum2")(add) + sum_op3 = operation(name="sum_op3", needs=["sum1", "sum2"], provides="sum3")(add) + net = compose(name="test_net")(sum_op1, sum_op2, sum_op3) - results = net({'sum1': sum1, 'sum2': sum2}) + results = net({"sum1": sum1, "sum2": sum2}) # Make sure we got expected result without having to pass a or b. - assert 'sum3' in results - assert results['sum3'] == add(sum1, sum2) + assert "sum3" in results + assert results["sum3"] == add(sum1, sum2) def test_output_based_pruning(): @@ -133,16 +223,16 @@ def test_output_based_pruning(): # Set up a network such that we don't need to provide a or b if we only # request sum3 as output. - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['c', 'd'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['c', 'sum2'], provides='sum3')(add) - net = compose(name='test_net')(sum_op1, sum_op2, sum_op3) + sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add) + sum_op2 = operation(name="sum_op2", needs=["c", "d"], provides="sum2")(add) + sum_op3 = operation(name="sum_op3", needs=["c", "sum2"], provides="sum3")(add) + net = compose(name="test_net")(sum_op1, sum_op2, sum_op3) - results = net({'c': c, 'd': d}, outputs=['sum3']) + results = net({"a": 0, "b": 0, "c": c, "d": d}, outputs=["sum3"]) # Make sure we got expected result without having to pass a or b. - assert 'sum3' in results - assert results['sum3'] == add(c, add(c, d)) + assert "sum3" in results + assert results["sum3"] == add(c, add(c, d)) def test_input_output_based_pruning(): @@ -155,16 +245,16 @@ def test_input_output_based_pruning(): # Set up a network such that we don't need to provide a or b d if we only # request sum3 as output and if we provide sum2. - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['c', 'd'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['c', 'sum2'], provides='sum3')(add) - net = compose(name='test_net')(sum_op1, sum_op2, sum_op3) + sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add) + sum_op2 = operation(name="sum_op2", needs=["c", "d"], provides="sum2")(add) + sum_op3 = operation(name="sum_op3", needs=["c", "sum2"], provides="sum3")(add) + net = compose(name="test_net")(sum_op1, sum_op2, sum_op3) - results = net({'c': c, 'sum2': sum2}, outputs=['sum3']) + results = net({"c": c, "sum2": sum2}, outputs=["sum3"]) # Make sure we got expected result without having to pass a, b, or d. - assert 'sum3' in results - assert results['sum3'] == add(c, sum2) + assert "sum3" in results + assert results["sum3"] == add(c, sum2) def test_pruning_raises_for_bad_output(): @@ -173,15 +263,260 @@ def test_pruning_raises_for_bad_output(): # Set up a network that doesn't have the output sum4, which we'll request # later. - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['c', 'd'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['c', 'sum2'], provides='sum3')(add) - net = compose(name='test_net')(sum_op1, sum_op2, sum_op3) + sum_op1 = operation(name="sum_op1", needs=["a", "b"], provides="sum1")(add) + sum_op2 = operation(name="sum_op2", needs=["c", "d"], provides="sum2")(add) + sum_op3 = operation(name="sum_op3", needs=["c", "sum2"], provides="sum3")(add) + net = compose(name="test_net")(sum_op1, sum_op2, sum_op3) # Request two outputs we can compute and one we can't compute. Assert # that this raises a ValueError. - assert_raises(ValueError, net, {'a': 1, 'b': 2, 'c': 3, 'd': 4}, - outputs=['sum1', 'sum3', 'sum4']) + with pytest.raises(ValueError) as exinfo: + net({"a": 1, "b": 2, "c": 3, "d": 4}, outputs=["sum1", "sum3", "sum4"]) + assert exinfo.match("sum4") + + +def test_pruning_not_overrides_given_intermediate(): + # Test #25: v1.2.4 overwrites intermediate data when no output asked + pipeline = compose(name="pipeline")( + operation(name="not run", needs=["a"], provides=["overriden"])(scream), + operation(name="op", needs=["overriden", "c"], provides=["asked"])(add), + ) + + inputs = {"a": 5, "overriden": 1, "c": 2} + exp = {"a": 5, "overriden": 1, "c": 2, "asked": 3} + # v1.2.4.ok + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + # FAILs + # - on v1.2.4 with (overriden, asked): = (5, 7) instead of (1, 3) + # - on #18(unsatisfied) + #23(ordered-sets) with (overriden, asked) = (5, 7) instead of (1, 3) + # FIXED on #26 + assert pipeline(inputs) == exp + + ## Test OVERWITES + # + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + assert overwrites == {} # unjust must have been pruned + + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline(inputs) == exp + assert overwrites == {} # unjust must have been pruned + + ## Test Parallel + # + pipeline.set_execution_method("parallel") + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + # assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + assert overwrites == {} # unjust must have been pruned + + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline(inputs) == exp + assert overwrites == {} # unjust must have been pruned + + +def test_pruning_multiouts_not_override_intermediates1(): + # Test #25: v.1.2.4 overwrites intermediate data when a previous operation + # must run for its other outputs (outputs asked or not) + pipeline = compose(name="pipeline")( + operation(name="must run", needs=["a"], provides=["overriden", "calced"])( + lambda x: (x, 2 * x) + ), + operation(name="add", needs=["overriden", "calced"], provides=["asked"])(add), + ) + + inputs = {"a": 5, "overriden": 1, "c": 2} + exp = {"a": 5, "overriden": 1, "calced": 10, "asked": 11} + # FAILs + # - on v1.2.4 with (overriden, asked) = (5, 15) instead of (1, 11) + # - on #18(unsatisfied) + #23(ordered-sets) like v1.2.4. + # FIXED on #26 + assert pipeline({"a": 5, "overriden": 1}) == exp + # FAILs + # - on v1.2.4 with KeyError: 'e', + # - on #18(unsatisfied) + #23(ordered-sets) with empty result. + # FIXED on #26 + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + + ## Test OVERWITES + # + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline({"a": 5, "overriden": 1}) == exp + assert overwrites == {"overriden": 5} + + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + assert overwrites == {"overriden": 5} + + ## Test parallel + # + pipeline.set_execution_method("parallel") + assert pipeline({"a": 5, "overriden": 1}) == exp + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + + +@pytest.mark.xfail( + sys.version_info < (3, 6), + reason="PY3.5- have unstable dicts." + "E.g. https://travis-ci.org/ankostis/graphkit/jobs/595841023", +) +def test_pruning_multiouts_not_override_intermediates2(): + # Test #25: v.1.2.4 overrides intermediate data when a previous operation + # must run for its other outputs (outputs asked or not) + # SPURIOUS FAILS in < PY3.6 due to unordered dicts, + # eg https://travis-ci.org/ankostis/graphkit/jobs/594813119 + pipeline = compose(name="pipeline")( + operation(name="must run", needs=["a"], provides=["overriden", "e"])( + lambda x: (x, 2 * x) + ), + operation(name="op1", needs=["overriden", "c"], provides=["d"])(add), + operation(name="op2", needs=["d", "e"], provides=["asked"])(mul), + ) + + inputs = {"a": 5, "overriden": 1, "c": 2} + exp = {"a": 5, "overriden": 1, "c": 2, "d": 3, "e": 10, "asked": 30} + # FAILs + # - on v1.2.4 with (overriden, asked) = (5, 70) instead of (1, 13) + # - on #18(unsatisfied) + #23(ordered-sets) like v1.2.4. + # FIXED on #26 + assert pipeline(inputs) == exp + # FAILs + # - on v1.2.4 with KeyError: 'e', + # - on #18(unsatisfied) + #23(ordered-sets) with empty result. + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + # FIXED on #26 + + ## Test OVERWITES + # + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline(inputs) == exp + assert overwrites == {"overriden": 5} + + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + assert overwrites == {"overriden": 5} + + ## Test parallel + # + pipeline.set_execution_method("parallel") + assert pipeline(inputs) == exp + assert pipeline(inputs, ["asked"]) == filtdict(exp, "asked") + + +def test_pruning_with_given_intermediate_and_asked_out(): + # Test #24: v1.2.4 does not prune before given intermediate data when + # outputs not asked, but does so when output asked. + pipeline = compose(name="pipeline")( + operation(name="unjustly pruned", needs=["given-1"], provides=["a"])(identity), + operation(name="shortcuted", needs=["a", "b"], provides=["given-2"])(add), + operation(name="good_op", needs=["a", "given-2"], provides=["asked"])(add), + ) + + exp = {"given-1": 5, "b": 2, "given-2": 2, "a": 5, "asked": 7} + # v1.2.4 is ok + assert pipeline({"given-1": 5, "b": 2, "given-2": 2}) == exp + # FAILS + # - on v1.2.4 with KeyError: 'a', + # - on #18 (unsatisfied) with no result. + # FIXED on #18+#26 (new dag solver). + assert pipeline({"given-1": 5, "b": 2, "given-2": 2}, ["asked"]) == filtdict( + exp, "asked" + ) + + ## Test OVERWITES + # + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline({"given-1": 5, "b": 2, "given-2": 2}) == exp + assert overwrites == {} + + overwrites = {} + pipeline.set_overwrites_collector(overwrites) + assert pipeline({"given-1": 5, "b": 2, "given-2": 2}, ["asked"]) == filtdict( + exp, "asked" + ) + assert overwrites == {} + + ## Test parallel + # FAIL! in #26! + # + pipeline.set_execution_method("parallel") + assert pipeline({"given-1": 5, "b": 2, "given-2": 2}) == exp + assert pipeline({"given-1": 5, "b": 2, "given-2": 2}, ["asked"]) == filtdict( + exp, "asked" + ) + + +def test_unsatisfied_operations(): + # Test that operations with partial inputs are culled and not failing. + pipeline = compose(name="pipeline")( + operation(name="add", needs=["a", "b1"], provides=["a+b1"])(add), + operation(name="sub", needs=["a", "b2"], provides=["a-b2"])(sub), + ) + + exp = {"a": 10, "b1": 2, "a+b1": 12} + assert pipeline({"a": 10, "b1": 2}) == exp + assert pipeline({"a": 10, "b1": 2}, outputs=["a+b1"]) == filtdict(exp, "a+b1") + + exp = {"a": 10, "b2": 2, "a-b2": 8} + assert pipeline({"a": 10, "b2": 2}) == exp + assert pipeline({"a": 10, "b2": 2}, outputs=["a-b2"]) == filtdict(exp, "a-b2") + + ## Test parallel + # + pipeline.set_execution_method("parallel") + exp = {"a": 10, "b1": 2, "a+b1": 12} + assert pipeline({"a": 10, "b1": 2}) == exp + assert pipeline({"a": 10, "b1": 2}, outputs=["a+b1"]) == filtdict(exp, "a+b1") + + exp = {"a": 10, "b2": 2, "a-b2": 8} + assert pipeline({"a": 10, "b2": 2}) == exp + assert pipeline({"a": 10, "b2": 2}, outputs=["a-b2"]) == filtdict(exp, "a-b2") + + +def test_unsatisfied_operations_same_out(): + # Test unsatisfied pairs of operations providing the same output. + pipeline = compose(name="pipeline")( + operation(name="mul", needs=["a", "b1"], provides=["ab"])(mul), + operation(name="div", needs=["a", "b2"], provides=["ab"])(floordiv), + operation(name="add", needs=["ab", "c"], provides=["ab_plus_c"])(add), + ) + + exp = {"a": 10, "b1": 2, "c": 1, "ab": 20, "ab_plus_c": 21} + assert pipeline({"a": 10, "b1": 2, "c": 1}) == exp + assert pipeline({"a": 10, "b1": 2, "c": 1}, outputs=["ab_plus_c"]) == filtdict( + exp, "ab_plus_c" + ) + + exp = {"a": 10, "b2": 2, "c": 1, "ab": 5, "ab_plus_c": 6} + assert pipeline({"a": 10, "b2": 2, "c": 1}) == exp + assert pipeline({"a": 10, "b2": 2, "c": 1}, outputs=["ab_plus_c"]) == filtdict( + exp, "ab_plus_c" + ) + + ## Test parallel + # + # FAIL! in #26 + pipeline.set_execution_method("parallel") + exp = {"a": 10, "b1": 2, "c": 1, "ab": 20, "ab_plus_c": 21} + assert pipeline({"a": 10, "b1": 2, "c": 1}) == exp + assert pipeline({"a": 10, "b1": 2, "c": 1}, outputs=["ab_plus_c"]) == filtdict( + exp, "ab_plus_c" + ) + # + # FAIL! in #26 + exp = {"a": 10, "b2": 2, "c": 1, "ab": 5, "ab_plus_c": 6} + assert pipeline({"a": 10, "b2": 2, "c": 1}) == exp + assert pipeline({"a": 10, "b2": 2, "c": 1}, outputs=["ab_plus_c"]) == filtdict( + exp, "ab_plus_c" + ) def test_optional(): @@ -191,21 +526,132 @@ def test_optional(): def addplusplus(a, b, c=0): return a + b + c - sum_op = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum')(addplusplus) + sum_op = operation( + name="sum_op1", needs=["a", "b", modifiers.optional("c")], provides="sum" + )(addplusplus) - net = compose(name='test_net')(sum_op) + net = compose(name="test_net")(sum_op) # Make sure output with optional arg is as expected. - named_inputs = {'a': 4, 'b': 3, 'c': 2} + named_inputs = {"a": 4, "b": 3, "c": 2} results = net(named_inputs) - assert 'sum' in results - assert results['sum'] == sum(named_inputs.values()) + assert "sum" in results + assert results["sum"] == sum(named_inputs.values()) # Make sure output without optional arg is as expected. - named_inputs = {'a': 4, 'b': 3} + named_inputs = {"a": 4, "b": 3} results = net(named_inputs) - assert 'sum' in results - assert results['sum'] == sum(named_inputs.values()) + assert "sum" in results + assert results["sum"] == sum(named_inputs.values()) + + +def test_sideffects(): + # Function without return value. + def extend(box): + box.extend([1, 2]) + + def increment(box): + for i in range(len(box)): + box[i] += 1 + + # Designate `a`, `b` as sideffect inp/out arguments. + graph = compose("mygraph")( + operation( + name="extend", + needs=["box", modifiers.sideffect("a")], + provides=[modifiers.sideffect("b")], + )(extend), + operation( + name="increment", + needs=["box", modifiers.sideffect("b")], + provides=modifiers.sideffect("c"), + )(increment), + ) + + assert graph({"box": [0], "a": True})["box"] == [1, 2, 3] + + # Reverse order of functions. + graph = compose("mygraph")( + operation( + name="increment", + needs=["box", modifiers.sideffect("a")], + provides=modifiers.sideffect("b"), + )(increment), + operation( + name="extend", + needs=["box", modifiers.sideffect("b")], + provides=[modifiers.sideffect("c")], + )(extend), + ) + + assert graph({"box": [0], "a": None})["box"] == [1, 1, 2] + + +@pytest.mark.xfail( + sys.version_info < (3, 6), + reason="PY3.5- have unstable dicts." + "E.g. https://travis-ci.org/ankostis/graphkit/jobs/595793872", +) +def test_optional_per_function_with_same_output(): + # Test that the same need can be both optional and not on different operations. + # + ## ATTENTION, the selected function is NOT the one with more inputs + # but the 1st satisfiable function added in the network. + + add_op = operation(name="add", needs=["a", "b"], provides="a+-b")(add) + sub_op_optional = operation( + name="sub_opt", needs=["a", modifiers.optional("b")], provides="a+-b" + )(lambda a, b=10: a - b) + + # Normal order + # + pipeline = compose(name="partial_optionals")(add_op, sub_op_optional) + # + named_inputs = {"a": 1, "b": 2} + assert pipeline(named_inputs) == {"a": 1, "a+-b": 3, "b": 2} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": 3} + # + named_inputs = {"a": 1} + assert pipeline(named_inputs) == {"a": 1, "a+-b": -9} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": -9} + + # Inverse op order + # + pipeline = compose(name="partial_optionals")(sub_op_optional, add_op) + # + named_inputs = {"a": 1, "b": 2} + assert pipeline(named_inputs) == {"a": 1, "a+-b": -1, "b": 2} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": -1} + # + named_inputs = {"a": 1} + assert pipeline(named_inputs) == {"a": 1, "a+-b": -9} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": -9} + + # PARALLEL + Normal order + # + pipeline = compose(name="partial_optionals")(add_op, sub_op_optional) + pipeline.set_execution_method("parallel") + # + named_inputs = {"a": 1, "b": 2} + assert pipeline(named_inputs) == {"a": 1, "a+-b": 3, "b": 2} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": 3} + # + named_inputs = {"a": 1} + assert pipeline(named_inputs) == {"a": 1, "a+-b": -9} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": -9} + + # PARALLEL + Inverse op order + # + pipeline = compose(name="partial_optionals")(sub_op_optional, add_op) + pipeline.set_execution_method("parallel") + # + named_inputs = {"a": 1, "b": 2} + assert pipeline(named_inputs) == {"a": 1, "a+-b": -1, "b": 2} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": -1} + # + named_inputs = {"a": 1} + assert pipeline(named_inputs) == {"a": 1, "a+-b": -9} + assert pipeline(named_inputs, ["a+-b"]) == {"a+-b": -9} def test_deleted_optional(): @@ -217,52 +663,100 @@ def addplusplus(a, b, c=0): return a + b + c # Here, a DeleteInstruction will be inserted for the optional need 'c'. - sum_op1 = operation(name='sum_op1', needs=['a', 'b', modifiers.optional('c')], provides='sum1')(addplusplus) - sum_op2 = operation(name='sum_op2', needs=['sum1', 'sum1'], provides='sum2')(add) - net = compose(name='test_net')(sum_op1, sum_op2) + sum_op1 = operation( + name="sum_op1", needs=["a", "b", modifiers.optional("c")], provides="sum1" + )(addplusplus) + sum_op2 = operation(name="sum_op2", needs=["sum1", "sum1"], provides="sum2")(add) + net = compose(name="test_net")(sum_op1, sum_op2) # DeleteInstructions are used only when a subset of outputs are requested. - results = net({'a': 4, 'b': 3}, outputs=['sum2']) - assert 'sum2' in results + results = net({"a": 4, "b": 3}, outputs=["sum2"]) + assert "sum2" in results + + +def test_deleteinstructs_vary_with_inputs(): + # Check #21: DeleteInstructions positions vary when inputs change. + def count_deletions(steps): + return sum(isinstance(n, DeleteInstruction) for n in steps) + + pipeline = compose(name="pipeline")( + operation(name="a free without b", needs=["a"], provides=["aa"])(identity), + operation(name="satisfiable", needs=["a", "b"], provides=["ab"])(add), + operation( + name="optional ab", + needs=["aa", modifiers.optional("ab")], + provides=["asked"], + )(lambda a, ab=10: a + ab), + ) + inp = {"a": 2, "b": 3} + exp = inp.copy() + exp.update({"aa": 2, "ab": 5, "asked": 7}) + res = pipeline(inp) + assert res == exp # ok + steps11 = pipeline.compile(inp).steps + res = pipeline(inp, outputs=["asked"]) + assert res == filtdict(exp, "asked") # ok + steps12 = pipeline.compile(inp, ["asked"]).steps + + inp = {"a": 2} + exp = inp.copy() + exp.update({"aa": 2, "asked": 12}) + res = pipeline(inp) + assert res == exp # ok + steps21 = pipeline.compile(inp).steps + res = pipeline(inp, outputs=["asked"]) + assert res == filtdict(exp, "asked") # ok + steps22 = pipeline.compile(inp, ["asked"]).steps + + # When no outs, no del-instructs. + assert steps11 != steps12 + assert count_deletions(steps11) == 0 + assert steps21 != steps22 + assert count_deletions(steps21) == 0 + + # Check steps vary with inputs + # + # FAILs in v1.2.4 + #18, PASS in #26 + assert steps11 != steps21 + + # Check deletes vary with inputs + # + # FAILs in v1.2.4 + #18, PASS in #26 + assert count_deletions(steps12) != count_deletions(steps22) +@pytest.mark.slow def test_parallel_execution(): import time + delay = 0.5 + def fn(x): - time.sleep(1) + time.sleep(delay) print("fn %s" % (time.time() - t0)) return 1 + x - def fn2(a,b): - time.sleep(1) + def fn2(a, b): + time.sleep(delay) print("fn2 %s" % (time.time() - t0)) - return a+b + return a + b def fn3(z, k=1): - time.sleep(1) + time.sleep(delay) print("fn3 %s" % (time.time() - t0)) return z + k pipeline = compose(name="l", merge=True)( - # the following should execute in parallel under threaded execution mode operation(name="a", needs="x", provides="ao")(fn), operation(name="b", needs="x", provides="bo")(fn), - # this should execute after a and b have finished operation(name="c", needs=["ao", "bo"], provides="co")(fn2), - - operation(name="d", - needs=["ao", modifiers.optional("k")], - provides="do")(fn3), - + operation(name="d", needs=["ao", modifiers.optional("k")], provides="do")(fn3), operation(name="e", needs=["ao", "bo"], provides="eo")(fn2), operation(name="f", needs="eo", provides="fo")(fn), - operation(name="g", needs="fo", provides="go")(fn) - - + operation(name="g", needs="fo", provides="go")(fn), ) t0 = time.time() @@ -280,38 +774,43 @@ def fn3(z, k=1): # make sure results are the same using either method assert result_sequential == result_threaded + +@pytest.mark.slow def test_multi_threading(): import time import random from multiprocessing.dummy import Pool def op_a(a, b): - time.sleep(random.random()*.02) - return a+b + time.sleep(random.random() * 0.02) + return a + b def op_b(c, b): - time.sleep(random.random()*.02) - return c+b + time.sleep(random.random() * 0.02) + return c + b def op_c(a, b): - time.sleep(random.random()*.02) - return a*b + time.sleep(random.random() * 0.02) + return a * b pipeline = compose(name="pipeline", merge=True)( - operation(name="op_a", needs=['a', 'b'], provides='c')(op_a), - operation(name="op_b", needs=['c', 'b'], provides='d')(op_b), - operation(name="op_c", needs=['a', 'b'], provides='e')(op_c), + operation(name="op_a", needs=["a", "b"], provides="c")(op_a), + operation(name="op_b", needs=["c", "b"], provides="d")(op_b), + operation(name="op_c", needs=["a", "b"], provides="e")(op_c), ) def infer(i): # data = open("616039-bradpitt.jpg").read() outputs = ["c", "d", "e"] - results = pipeline({"a": 1, "b":2}, outputs) - assert tuple(sorted(results.keys())) == tuple(sorted(outputs)), (outputs, results) + results = pipeline({"a": 1, "b": 2}, outputs) + assert tuple(sorted(results.keys())) == tuple(sorted(outputs)), ( + outputs, + results, + ) return results - N = 100 - for i in range(20, 200): + N = 33 + for i in range(13, 61): pool = Pool(i) pool.map(infer, range(N)) pool.close() @@ -325,56 +824,44 @@ def infer(i): # We first define some basic operations class Sum(Operation): - def compute(self, inputs): a = inputs[0] b = inputs[1] - return [a+b] + return [a + b] class Mul(Operation): - def compute(self, inputs): a = inputs[0] b = inputs[1] - return [a*b] + return [a * b] # This is an example of an operation that takes a parameter. # It also illustrates an operation that returns multiple outputs class Pow(Operation): - def compute(self, inputs): a = inputs[0] outputs = [] - for y in range(1, self.params['exponent']+1): + for y in range(1, self.params["exponent"] + 1): p = math.pow(a, y) outputs.append(p) return outputs + def test_backwards_compatibility(): - sum_op1 = Sum( - name="sum_op1", - provides=["sum_ab"], - needs=["a", "b"] - ) - mul_op1 = Mul( - name="mul_op1", - provides=["sum_ab_times_b"], - needs=["sum_ab", "b"] - ) + sum_op1 = Sum(name="sum_op1", provides=["sum_ab"], needs=["a", "b"]) + mul_op1 = Mul(name="mul_op1", provides=["sum_ab_times_b"], needs=["sum_ab", "b"]) pow_op1 = Pow( name="pow_op1", needs=["sum_ab"], provides=["sum_ab_p1", "sum_ab_p2", "sum_ab_p3"], - params={"exponent": 3} + params={"exponent": 3}, ) sum_op2 = Sum( - name="sum_op2", - provides=["p1_plus_p2"], - needs=["sum_ab_p1", "sum_ab_p2"], + name="sum_op2", provides=["p1_plus_p2"], needs=["sum_ab_p1", "sum_ab_p2"] ) net = network.Network() @@ -392,10 +879,25 @@ def test_backwards_compatibility(): # # get all outputs - pprint(net.compute(outputs=None, named_inputs={'a': 1, 'b': 2})) + exp = { + "a": 1, + "b": 2, + "p1_plus_p2": 12.0, + "sum_ab": 3, + "sum_ab_p1": 3.0, + "sum_ab_p2": 9.0, + "sum_ab_p3": 27.0, + "sum_ab_times_b": 6, + } + assert net.compute(outputs=None, named_inputs={"a": 1, "b": 2}) == exp # get specific outputs - pprint(net.compute(outputs=["sum_ab_times_b"], named_inputs={'a': 1, 'b': 2})) + exp = {"sum_ab_times_b": 6} + assert net.compute(outputs=["sum_ab_times_b"], named_inputs={"a": 1, "b": 2}) == exp # start with inputs already computed - pprint(net.compute(outputs=["sum_ab_times_b"], named_inputs={"sum_ab": 1, "b": 2})) + exp = {"sum_ab_times_b": 2} + assert ( + net.compute(outputs=["sum_ab_times_b"], named_inputs={"sum_ab": 1, "b": 2}) + == exp + ) diff --git a/test/test_plot.py b/test/test_plot.py new file mode 100644 index 00000000..d17201cb --- /dev/null +++ b/test/test_plot.py @@ -0,0 +1,190 @@ +# Copyright 2016, Yahoo Inc. +# Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. + +import sys +from operator import add + +import pytest + +from graphkit import base, compose, network, operation, plot +from graphkit.modifiers import optional + + +@pytest.fixture +def pipeline(): + return compose(name="netop")( + operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), + operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])( + lambda a, b=1: a - b + ), + operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), + ) + + +@pytest.fixture(params=[{"a": 1}, {"a": 1, "b1": 2}]) +def inputs(request): + return {"a": 1, "b1": 2} + + +@pytest.fixture(params=[None, ("a", "b1")]) +def input_names(request): + return request.param + + +@pytest.fixture(params=[None, ["asked", "b1"]]) +def outputs(request): + return request.param + + +@pytest.fixture(params=[None, 1]) +def solution(pipeline, inputs, outputs, request): + return request.param and pipeline(inputs, outputs) + + +###### TEST CASES ####### +## + + +def test_plotting_docstring(): + common_formats = ".png .dot .jpg .jpeg .pdf .svg".split() + for ext in common_formats: + assert ext in base.NetworkOperation.plot.__doc__ + assert ext in network.Network.plot.__doc__ + + +@pytest.mark.slow +def test_plot_formats(pipeline, tmp_path): + ## Generate all formats (not needing to save files) + + # run it here (and not in ficture) to ansure `last_plan` exists. + inputs = {"a": 1, "b1": 2} + outputs = ["asked", "b1"] + solution = pipeline(inputs, outputs) + + # The 1st list does not working on my PC, or travis. + # NOTE: maintain the other lists manually from the Exception message. + failing_formats = ".dia .hpgl .mif .mp .pcl .pic .vtx .xlib".split() + # The subsequent format names producing the same dot-file. + dupe_formats = [ + ".cmapx_np", # .cmapx + ".imap_np", # .imap + ".jpeg", # .jpe + ".jpg", # .jpe + ".plain-ext", # .plain + ] + null_formats = ".cmap .ismap".split() + forbidden_formats = set(failing_formats + dupe_formats + null_formats) + formats_to_check = sorted(set(plot.supported_plot_formats()) - forbidden_formats) + + # Collect old dots to detect dupes. + prev_renders = {} + dupe_errs = [] + for ext in formats_to_check: + # Check Network. + # + render = pipeline.plot(solution=solution).create(format=ext[1:]) + if not render: + dupe_errs.append("\n null: %s" % ext) + + elif render in prev_renders.values(): + dupe_errs.append( + "\n dupe: %s <--> %s" + % (ext, [pext for pext, pdot in prev_renders.items() if pdot == render]) + ) + else: + prev_renders[ext] = render + + if dupe_errs: + raise AssertionError("Failed pydot formats: %s" % "".join(sorted(dupe_errs))) + + +def test_plotters_hierarchy(pipeline, inputs, outputs): + # Plotting original network, no plan. + base_dot = str(pipeline.plot(inputs=inputs, outputs=outputs)) + assert base_dot + assert pipeline.name in str(base_dot) + + solution = pipeline(inputs, outputs) + + # Plotting delegates to netwrok plan. + plan_dot = str(pipeline.plot(inputs=inputs, outputs=outputs)) + assert plan_dot + assert plan_dot != base_dot + assert pipeline.name in str(plan_dot) + + # Plot a plan + solution, which must be different from all before. + sol_plan_dot = str(pipeline.plot(inputs=inputs, outputs=outputs, solution=solution)) + assert sol_plan_dot != base_dot + assert sol_plan_dot != plan_dot + assert pipeline.name in str(plan_dot) + + plan = pipeline.net.last_plan + pipeline.net.last_plan = None + + # We resetted last_plan to check if it reproduces original. + base_dot2 = str(pipeline.plot(inputs=inputs, outputs=outputs)) + assert str(base_dot2) == str(base_dot) + + # Calling plot directly on plan misses netop.name + raw_plan_dot = str(plan.plot(inputs=inputs, outputs=outputs)) + assert pipeline.name not in str(raw_plan_dot) + + # Chek plan does not contain solution, unless given. + raw_sol_plan_dot = str(plan.plot(inputs=inputs, outputs=outputs, solution=solution)) + assert raw_sol_plan_dot != raw_plan_dot + + +def test_plot_bad_format(pipeline, tmp_path): + with pytest.raises(ValueError, match="Unknown file format") as exinfo: + pipeline.plot(filename="bad.format") + + ## Check help msg lists all siupported formats + for ext in plot.supported_plot_formats(): + assert exinfo.match(ext) + + +def test_plot_write_file(pipeline, tmp_path): + # Try saving a file from one format. + + fpath = tmp_path / "network.png" + dot1 = pipeline.plot(str(fpath)) + assert fpath.exists() + assert dot1 + + +def _check_plt_img(img): + assert img is not None + assert len(img) > 0 + + +def test_plot_matpotlib(pipeline, tmp_path): + ## Try matplotlib Window, but # without opening a Window. + + if sys.version_info < (3, 5): + # On PY< 3.5 it fails with: + # nose.proxy.TclError: no display name and no $DISPLAY environment variable + # eg https://travis-ci.org/ankostis/graphkit/jobs/593957996 + import matplotlib + + matplotlib.use("Agg") + # do not open window in headless travis + img = pipeline.plot(show=-1) + _check_plt_img(img) + + +def test_plot_jupyter(pipeline, tmp_path): + ## Try returned Jupyter SVG. + + dot = pipeline.plot() + s = dot._repr_svg_() + assert "SVG" in s + + +def test_plot_legend(pipeline, tmp_path): + ## Try returned Jupyter SVG. + + dot = plot.legend() + assert dot + + img = plot.legend(show=-1) + _check_plt_img(img)