Merge branch 'master' into validate-multiple-instances

bindings/python/dlite-path-python.i

@@ -37,12 +37,31 @@
 
 
 %pythoncode %{
-storage_path = FUPath("storages")
-storage_plugin_path = FUPath("storage-plugins")
-mapping_plugin_path = FUPath("mapping-plugins")
-python_storage_plugin_path = FUPath("python-storage-plugins")
-python_mapping_plugin_path = FUPath("python-mapping-plugins")
-python_protocol_plugin_path = FUPath("python-protocol-plugins")
+import sys
+from importlib.metadata import entry_points
+
+def _create_path(name):
+    """Return new DLite search path object, with given name."""
+    if sys.version_info < (3, 10):  # Fallback for Python < 3.10
+        eps = entry_points().get(f"dlite.{name}", ())
+    else:  # For Python 3.10+
+        eps = entry_points(group=f"dlite.{name}")
+
+    path = FUPath(name)
+    path.name = name
+    for entry_point in eps:
+        path.append(entry_point.value)
+    return path
+
+# Create DLite search paths objects
+storage_path = _create_path("storages")
+template_path = _create_path("templates")
+storage_plugin_path = _create_path("storage-plugins")
+mapping_plugin_path = _create_path("mapping-plugins")
+python_storage_plugin_path = _create_path("python-storage-plugins")
+python_mapping_plugin_path = _create_path("python-mapping-plugins")
+python_protocol_plugin_path = _create_path("python-protocol-plugins")
+
 
 # Update default search paths
 from pathlib import Path

bindings/python/dlite-path.i

@@ -24,6 +24,7 @@
   //#include "utils/compat.h"
 #include "utils/fileutils.h"
   //#include "dlite-macros.h"
+#include "dlite-codegen.h"
 #include "pyembed/dlite-python-storage.h"
 #include "pyembed/dlite-python-mapping.h"
 #include "pyembed/dlite-python-protocol.h"
@@ -49,6 +50,8 @@ Creates a _Path instance of type `pathtype`.
   _FUPaths(const char *pathtype) {
     if (strcmp(pathtype, "storages") == 0) {
       return dlite_storage_paths();
+    } else if (strcmp(pathtype, "templates") == 0) {
+      return dlite_codegen_path_get();
     } else if (strcmp(pathtype, "storage-plugins") == 0) {
       return dlite_storage_plugin_paths_get();
     } else if (strcmp(pathtype, "mapping-plugins") == 0) {

bindings/python/scripts/dlite-validate

@@ -7,8 +7,6 @@ import json
 import re
 from pathlib import Path
 
-import dlite
-
 
 def parse(url, driver=None, options="mode=r", id=None):
     """Loads an instance from storage.
@@ -23,6 +21,8 @@ def parse(url, driver=None, options="mode=r", id=None):
     Returns:
         List of new instances.
     """
+    import dlite
+
     loc = url.split("#", 1)[0].split("?",1)[0]
     if driver is None:
         driver = Path(loc).suffix.lstrip('.').lower()
@@ -115,6 +115,15 @@ def main():
 
     args = parser.parse_args()
 
+    if not args.debug:
+        # Turn off DLite warnings about behavior changes
+        import __main__
+        __main__.DLITE_BEHAVIOR = True
+
+    # Delay importing dlite to after options are processed such that it is
+    # possible to configure DLite by settings variables in __main__.
+    import dlite
+
     for path in args.storage_path:
         dlite.storage_path.append(Path(path).resolve())
 

doc/user_guide/index.rst

@@ -13,6 +13,7 @@ User Guide
    storage_plugins
    storage_plugins_mongodb
    protocol_plugins
+   search_paths
    mappings
    transactions
    tools

doc/user_guide/search_paths.md

@@ -0,0 +1,103 @@
+Search paths
+============
+It is possible to extend DLite with new datamodels, code generation templates and plugins, by appending to corresponding search path.  The table below lists the different types of search paths that are available in DLite.
+
+| Search path type        | Description                                                    |
+|-------------------------|----------------------------------------------------------------|
+| storages                | Storage URLs or directory paths to datamodels                  |
+| templates               | Directory paths to code generator templates                    |
+| storage_plugins         | Directory paths to storage plugins (drivers) written in C      |
+| mapping-plugins         | Directory paths to mapping plugins written in C                |
+| python-storage-plugins  | Directory paths to storage plugins (drivers) written in Python |
+| python-mapping-plugins  | Directory paths to mapping plugins written in Python           |
+| python-protocol-plugins | Directory paths to protocol plugins written in Python          |
+
+Search paths can be extended in different three ways:
+* setting [environment variables]
+* appending to DLite path variables (from Python)
+* using [entry points] (only in user-defined Python packages)
+
+The table below lists the lists the name of the environment variables and Python path variables corresponding to the different types of search paths.
+
+| Search path type        | Environment variable name         | Python variable name              |
+|-------------------------|-----------------------------------|-----------------------------------|
+| storages                | DLITE_STORAGES                    | dlite.storage_path                |
+| templates               | DLITE_TEMPLATE_DIRS               | dlite.template_path               |
+| storage_plugins         | DLITE_STORAGE_PLUGIN_DIRS         | dlite.storage_plugin_path         |
+| mapping-plugins         | DLITE_MAPPING_PLUGIN_DIRS         | dlite.mapping-plugin_path         |
+| python-storage-plugins  | DLITE_PYTHON_STORAGE_PLUGIN_DIRS  | dlite.python-storage-plugin_path  |
+| python-mapping-plugins  | DLITE_PYTHON_MAPPING_PLUGIN_DIRS  | dlite.python-mapping-plugin_path  |
+| python-protocol-plugins | DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS | dlite.python-protocol-plugin_path |
+
+
+Setting environment variables
+-----------------------------
+This is typically done where you set up your environment, like in a virtualenv activate script or the users `~/.bashrc` file.
+
+:::{note}
+All the path variables, except for `DLITE_STORAGES`, uses (`:`) colon as path separator.
+However, since colon may appears in URLs, `DLITE_STORAGES` uses instead the pipe symbol (`|`) as path separator.
+:::
+
+See [environment variables] for more details.
+
+
+Appending to DLite path variables
+---------------------------------
+The `dlite` Python module defines the path variables listed in the table above.
+A Python script or application can configure new datamodels and plugins by appending to these variables.
+
+:::{example}
+Adding the sub-directory `datamodels` to the search path for datamodels, can be done with:
+
+```python
+>>> from pathlib import Path
+>>> import dlite
+>>> dlite.storage_path.append(Path("my_package_root") / "datamodels")
+```
+:::
+
+
+Using entry points
+------------------
+How to use entry points is easiest described with an example.
+Let us assume you have a package with the following directory layout:
+
+```
+project_root
+├── mypackage
+│   ├── __init__.py
+│   ├── mymodule.py
+│   ├── data
+│   │   ├── python_storage_plugins
+│   │   │   ├── myplugin.py
+│   │   │   └── anotherplugin.py
+│   │   └── datamodels
+│   │       ├── mydatamodel.yaml
+│   │       └── anotherdatamodel.yaml
+├── pyproject.toml
+├── README.md
+└── LICENSE
+```
+
+To make your datamodels and Python storage plugins available for users of your package, you can add the following section to your `pyproject.toml`:
+
+```toml
+[tool.setuptools.package-data]
+"mypackages.data.datamodels" = ["*.json", "*.yaml"]
+"mypackages.data.python_storage_plugins" = ["*.py"]
+
+# Note the quotes around dlite.python_storage_plugins to escape the embedded dot
+[project.entry-points."dlite.python_storage_plugins"]
+storage_plugin_path = "mypackage:data/python_storage_plugins"
+
+[project.entry-points."dlite.storages"]
+datamodel_path = "mypackage:data/datamodels"
+```
+
+See the [Setuptools documentation] for how to this can be done with `setup.py` or `setup.cfg`.
+
+
+[environment variables]: https://sintef.github.io/dlite/user_guide/environment_variables.html
+[entry points]: https://setuptools.pypa.io/en/latest/userguide/entry_point.html
+[Setuptools documentation]: https://setuptools.pypa.io/en/latest/userguide/index.html

doc/user_guide/storage_plugins.md

@@ -45,15 +45,15 @@ import dlite
 dlite.python_storage_plugin_path.append("/path/to/plugins/dir")
 ```
 
-Often drivers are connected to very specific datamodel (entities). 
+Often drivers are connected to very specific datamodel (entities).
 DLite will find these datamodels if the path to their directory is set with the
 environment variable `DLITE_STORAGES` or added within python with `dlite.storage_path.append` similarly to described above for drivers.
 
 
 ```{attention}
-Often, during development dlite will fail unexpectedly. This is typically either because of an error in the 
-datamodel or the driver. 
-The variable DLITE_PYDEBUG can be set as `export DLITE_PYDEBUG=` to get python debugging information. 
+Often, during development dlite will fail unexpectedly. This is typically either because of an error in the
+datamodel or the driver.
+The variable DLITE_PYDEBUG can be set as `export DLITE_PYDEBUG=` to get python debugging information.
 This will give information about the driver.
 It is advisable to first check that the datamodel is valid with the command `dlite-validate datamodelfilename`.
 ```
@@ -252,6 +252,21 @@ This could lead to confusing and hard-to-find bugs due to interference between y
 However, since DLite v0.5.23, plugins are evaluated in separate scopes (which are available in `dlite._plugindict).
 
 
+### Distributing storage plugins
+If you created a python package with new storage plugins, you can make them
+available for users of your package by creating entry points with the paths to the storage plugins.
+
+Add the following section to your `pyproject.toml`:
+
+```toml
+[project.entry-points.dlite]
+python_storage_plugins = "path/to/python_storage_plugins/directory"
+
+```
+
+See [Search paths] for more information about entry points.
+
+
 
 Working with storages from C and Fortran
 ----------------------------------------
@@ -271,3 +286,4 @@ An example is available in [ex4].
 [Python storage plugin example]: https://github.com/SINTEF/dlite/tree/master/examples/storage_plugin
 [ex1]: https://github.com/SINTEF/dlite/tree/master/examples/ex1
 [ex4]: https://github.com/SINTEF/dlite/tree/master/examples/ex4
+[Search paths]: https://sintef.github.io/dlite/user_guide/search_paths.html

src/config-paths.h.in

@@ -23,5 +23,12 @@
 #define dlite_TEMPLATES               "@dlite_TEMPLATES@"
 #define dlite_STORAGES                "@dlite_STORAGES@"
 
+/* dlite-pyembed.c does not imports config.h to avoid conflicts with Python .h
+   Include configurations needed by dlite-pyembed.c here */
+#cmakedefine HAVE_SETENV
+#cmakedefine HAVE__PUTENV_S
+#cmakedefine HAVE_UNSETENV
+
+
 
 #endif /* _DLITE_CONFIG_PATHS_H */

src/dlite-behavior.c

@@ -128,7 +128,7 @@ int dlite_behavior_get(const char *name)
     const char *ver = dlite_get_version();  // current version
     b->value = (strcmp_semver(ver, b->version_new) >= 0) ? 1 : 0;
 
-    dlite_warnx("Behavior `%s` is not configured. "
+    dlite_warnx("Behavior change `%s` is not configured. "
                 "It will be enabled by default from v%s. "
                 "See https://sintef.github.io/dlite/user_guide/configure_behavior_changes.html for more info.",
                 b->name, b->version_new);

src/dlite-codegen.h

@@ -10,6 +10,7 @@
   typically is code that will be written to file and compiled.
 */
 
+#include "utils/tgen.h"
 #include "utils/fileutils.h"