Merge pull request #18 from pspoerri/python_module_improvements

Python module improvements

CMakeLists.txt

@@ -71,6 +71,9 @@ include_directories( libs/gmock-gtest)
 include_directories( src)
 
 
+# Python installation path 
+set (PYTHON_PATH "python")
+
 # Build sources
 add_subdirectory( libs/libjson )
 add_subdirectory( libs/sha256 )
@@ -79,6 +82,7 @@ add_subdirectory( src )
 add_subdirectory( fortran )
 add_subdirectory( tools )
 add_subdirectory( unittest )
+add_subdirectory( python )
 
 file(APPEND ${TEST_SCRIPT} "echo \"RETURNING $res\" \n exit $res\n")
 file(INSTALL ${TEST_SCRIPT} DESTINATION ${CMAKE_BINARY_DIR}/install

README.md

@@ -18,7 +18,7 @@ As an example, porting scientific codes to graphical processing units, that requ
   * src/utils: logger and other C++ utilities
   * wrapper: a C wrapper that allows to interoperate the core C++ functionality from other languages like Fortran
 * doc/ : latex documentation, tutorials and presentations
-* python/ : contains various python tools based on the SerialBox format, namely
+* [python/](python/README.md) : contains various python tools based on the SerialBox format, namely
   * pp_ser: a tiny DSL/parser that facilitates inserting SerialBox statements into your Fortran Code. 
   * Visualizer: a python tool that helps visualizing serialized data
 * tools/ : various tools used to validation, dumping, converting or comparing of multiple serialized runs, etc. 

python/CMakeLists.txt

@@ -0,0 +1,4 @@
+INSTALL(DIRECTORY
+    "${CMAKE_SOURCE_DIR}/python/"
+    DESTINATION
+    "${PYTHON_PATH}/")

python/README.md

@@ -1,30 +1,91 @@
-# Python STELLA Serializer #
+# SerialBox Python Tools
 
-Currently only reads are supported.  
+This folder contains the SerialBox python tools. 
 
-## Using the Library
+- `pp_ser.py` - the SerialBox preproccessor to add the serialization directives to a Fortran program. For documentation see the docs folder. 
+- `serialbox` - the SerialBox python module that allows de-serializing and visualizing data stored by SerialBox
+- `test.py` - the unittests for the Python Tools
 
-### Reading
+## The `serialbox` Python Module
 
-Open the serialization:
+The serialbox python module makes use of the C++ SerialBox wrapper to read serialized data written by SerialBox. For convenience the required dynamic library is distributed with an installation of SerialBox. 
 
-    from Serializer import *
-    ser = Serializer("/scratch/olifu/pollen_case/test_pollen", "Error")
+### Installation
 
-and access the data with the exported dictionary:
+Build SerialBox as per the installation and building instructions. CMake installs the serialbox module to 
+`$CMAKE_INSTALL_PREFIX/python/serialbox` with `make install` in the build directory.
 
-    data = ser['TimeIntegratorUnittest.DoRKStage-out']['LargeTimeStep'][0]['RKStageNumber'][1]['pp']
+To use the serialbox python module either
 
-The interface allows removing the halo from the data:
+- Add `$CMAKE_INSTALL_PREFIX/python` to your `$PYTHONPATH`
+- cd to `$CMAKE_INSTALL_PREFIX/python` before executing any python scripts
+- Or copy `$CMAKE_INSTALL_PREFIX/python/serialbox` to your desired python modulepath
 
-    data = ser['TimeIntegratorUnittest.DoRKStage-out']['LargeTimeStep'][0]['RKStageNumber'][1]['pp', 'inner']
+### Reading and Visualizing Serialized Data
 
-### Visualizing
+In our example we want to look at serialized data from another process:
 
-Visualize the data:
-    from Visualizer import *
-    Visualizer(data, 'pp_in')
+```python
+datapath = "/scratch/jenkins/data/double/normal/"
+```
 
-## Creating the Shared Library
+#### Reading
 
-The shared library is directly created with CMAKE.
+First, we load the Serializer from the serialbox module. We obtain the data by initializing the `Serializer` by giving it the path to the data:
+
+```python
+from serialbox import Serializer
+ser = Serializer(datapath)
+```
+
+By default the Serializer loads the data stored in `Field.json`. If your data is stored with a different prefix, e.g. `Error.json`, then you can give the different prefix, e.g. `Error` as an additional argument to the serializer:
+
+```python
+err = Serializer(datapath, "Error")
+```
+
+In python the key-value format of the serializer is represented as a hash table. For example, if a savepoint is represented with the keys `TimeIntegratorUnittest.DoRKStage-out`, `LargeTimeStep=0` and `RKStageNumber=1` then this savepoint can be accessed as:
+
+```python
+savepoint =  ser['TimeIntegratorUnittest.DoRKStage-out']['LargeTimeStep'][0]['RKStageNumber'][1]
+```
+
+To then load the `pp` variable from the savepoint simply add `pp` to the hash table and we get:
+
+```python
+data = savepoint['pp']
+```
+
+`data` will now contain a numpy array that you can directly use with matplotlib and numpy. The interface allows removing the halo from the data by specifing `"inner"` as an additional argument:
+
+```python
+data = savepoint['pp', 'inner']
+```
+
+The serializer allows dynamic exploration of the data. The representation of the Serializer object returns only the stored savepoints.
+For our dataset the result executed in a ipython shell will look like this:
+
+```python
+In [4]: ser
+Out[4]: { 'FastWavesSCUnittest.UV-out' = [...], 'AdvectionPDBottUnittest.Init-in' = [...], 'AdvectionPDBottUnittest.DoTracers-out' = [...], 'ConvertTemperatureUnittest.DoT-out' = [...], 'SedimentationUnittest.DoTracers-out' = [...], 'VerticalDiffusionUnittest.PrepareStep-in' = [...], 'VerticalDiffusionUnittest.DoUVWT-in' = [...], 'RelaxationUnittest.Apply-out' = [...], 'ConvertTemperatureUnittest.DoTP-in' = [...], 'FastWavesSCUnittest.WPPTP-out' = [...], 'VerticalDiffusionUnittest.DoTracers-in' = [...], 'RelaxationUnittest.Apply-in' = [...], 'ConvertTemperatureUnittest.DoTP-out' = [...], 'FastWavesSCUnittest.DoSmallStep-in' = [...], 'TimeIntegratorUnittest.DoRKStage-out' = [...], 'HorizontalDiffusionUnittest.DoStep-out' = [...], 'VerticalAdvectionUnittest.DoUVW-in' = [...], 'AdvectionPDBottUnittest.RecalculateDensity-in' = [...], 'HorizontalDiffusionUnittest.ColdPool-in' = [...], 'FastWavesSCUnittest.ExplicitDivergence-out' = [...], 'HorizontalAdvectionUnittest.DoWWCon-out' = [...], 'FastWavesSCUnittest.LHS-in' = [...], 'FastWavesSCUnittest.AllSteps-in' = [...], 'SaturationAdjustmentUnittest.Apply-in' = [...], 'CoriolisUnittest.Apply-in' = [...], 'TimeIntegratorUnittest.DoRKStage-in' = [...], 'TimeIntegratorUnittest.DoStep-out' = [...], 'FastWavesSCUnittest.RHS-in' = [...], 'CoriolisUnittest.Apply-out' = [...], 'ConvertTemperatureUnittest.DoT-in' = [...], 'VerticalAdvectionUnittest.DoPPTP-in' = [...], 'FastWavesSCUnittest.UV-in' = [...], 'HorizontalDiffusionUnittest.ColdPool-out' = [...], 'FastWavesSCUnittest.Init-out' = [...], 'VerticalAdvectionUnittest.DoPPTP-out' = [...], 'HorizontalAdvectionUnittest.DoUV-in' = [...], 'TimeIntegratorUnittest.DoStep-in' = [...], 'FastWavesSCUnittest.WPPTP-in' = [...], 'ConstantFields' = [...], 'DycoreUnittest.DoStep-out' = [...], 'FastWavesSCUnittest.ExplicitDivergence-in' = [...], 'FastWavesSCUnittest.DoSmallStep-out' = [...], 'AdvectionPDBottUnittest.Init-out' = [...], 'VerticalAdvectionUnittest.DoUVW-out' = [...], 'AdvectionPDBottUnittest.DoTracers-in' = [...], 'HorizontalAdvectionUnittest.DoPPTP-out' = [...], 'FastWavesSCUnittest.AllSteps-out' = [...], 'AdvectionPDBottUnittest.RecalculateDensity-out' = [...], 'HorizontalAdvectionUnittest.DoWWCon-in' = [...], 'VerticalDiffusionUnittest.DoUVWT-out' = [...], 'HorizontalAdvectionUnittest.DoUV-out' = [...], 'FastWavesSCUnittest.Init-in' = [...], 'VerticalDiffusionUnittest.DoTracers-out' = [...], 'VerticalDiffusionUnittest.PrepareStep-out' = [...], 'FastWavesSCUnittest.RHS-out' = [...], 'FastWavesSCUnittest.LHS-out' = [...], 'HorizontalAdvectionUnittest.DoPPTP-in' = [...], 'HorizontalDiffusionUnittest.DoStep-in' = [...], 'DycoreUnittest.DoStep-in' = [...], 'SedimentationUnittest.DoTracers-in' = [...], 'SaturationAdjustmentUnittest.Apply-out' = [...] }
+```
+
+#### Visualizing
+
+To help visualizing the data serialbox contains a built-in Visualizer based on matplotlib. The visualizer expects two arguments: 
+
+- The 3D numpy array loaded by the serializer
+- The name of the plot
+
+In order to visualize the `pp` field stored in `data` from the code above it is sufficient to simply execute
+
+```python
+from serialbox import Visualizer
+Visualizer(data, 'pp')
+```
+
+This will open a window displaying the data:
+
+![Visualizer window](visualizer.png)
+
+Each k level is represented as a simple slice in the window.

python/Serializer.py → python/serialbox/Serializer.py

@@ -1,10 +1,10 @@
 #This file is released under terms of BSD license`
 #See LICENSE.txt for more information
 
-from .serialization import serializer
-
+"""Serializer module used to read serialized data"""
 
 class Savepoint(dict):
+    """Represents a serializer savepoint"""
     def __init__(self, ser, savepoint):
         self.savepoint = savepoint
         self.serializer = ser
@@ -74,7 +74,20 @@ def _nested_set(dic, keys, value):
     dic[keys[-1]] = value
 
 class Serializer(dict):
+    """The serializer that allows accessing serialized data""" 
     def __init__(self, directory=".", prefix="Field", openmode="r"):
+        """Create a serializer
+
+        Arguments:
+        directory -- The directory where the data is loaded
+        prefix -- The prefix of the serialized data
+        openmode -- The data mode: Anything other than `r` is currently not supported
+        """
+
+        if openmode != "r":
+            raise ValueError("Only reads are supported at the moment")
+
+        from .serialization import serializer
         self.serializer = serializer(directory, prefix, openmode)
         fnames = self.serializer.fieldnames
         finfos = self.serializer.fieldinfos

python/Visualizer.py → python/serialbox/Visualizer.py

@@ -1,6 +1,8 @@
 #This file is released under terms of BSD license`
 #See LICENSE.txt for more information
 
+"""Visualizer module used for visualization of serialized data."""
+
 import numpy as np
 from matplotlib import pyplot as plt
 from matplotlib.widgets import Slider, CheckButtons

python/__init__.py → python/serialbox/__init__.py

@@ -1,6 +1,6 @@
 #This file is released under terms of BSD license`
 #See LICENSE.txt for more information
-import matplotlib
+
 from .Visualizer import Visualizer
 from .Serializer import Serializer
 

python/serialization.py → python/serialbox/serialization.py

@@ -1,6 +1,10 @@
 #This file is released under terms of BSD license`
 #See LICENSE.txt for more information
 
+"""Internal serialization module that wraps the dynamic library of serialbox. 
+This module is used by the Serializer module.
+"""
+
 from __future__ import print_function
 import os, sys
 from os.path import join as pjoin
@@ -29,7 +33,6 @@
     # OS X
 
     libfile = pjoin(d, 'libSerialBox_Wrapper.'+library_postfix)
-    print("Trying path: {}".format(libfile))
     try:
         wrapper_try = ctypes.cdll.LoadLibrary(libfile)
         if wrapper_try is not None:

python/test.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#This file is released under terms of BSD license`
+#See LICENSE.txt for more information
+
+import unittest
+
+class Test(unittest.TestCase):
+	def test_load_serializer(self):
+		from serialbox import Serializer
+
+	def test_load_visualizer(self):
+		from serialbox import Visualizer
+
+if __name__ == '__main__':
+	unittest.main()
+

src/wrapper/CMakeLists.txt

@@ -14,13 +14,6 @@ add_library(
 target_link_libraries( SerialBoxWrapper SerialBox Utils json sha256 )
 install( TARGETS SerialBoxWrapper DESTINATION lib/ )
 
-set (PYTHON_PATH "python/serialbox")
-
-INSTALL(DIRECTORY
-    "${CMAKE_SOURCE_DIR}/python/"
-    DESTINATION
-    "${PYTHON_PATH}")
-
 add_library(SerialBox_Wrapper SHARED
     $<TARGET_OBJECTS:json_files>
     $<TARGET_OBJECTS:sha256_files>
@@ -29,4 +22,6 @@ add_library(SerialBox_Wrapper SHARED
     $<TARGET_OBJECTS:serialbox_wrapper_files>
 )
 install(TARGETS SerialBoxWrapper DESTINATION "lib")
-install(TARGETS SerialBox_Wrapper DESTINATION ${PYTHON_PATH})
+
+# The python library expects a serialbox dynamic library distributed with itself.
+install(TARGETS SerialBox_Wrapper DESTINATION ${PYTHON_PATH}/serialbox/)