Merge branch 'master' into 978-init-env-from-globals

bindings/python/dlite-collection-python.i

@@ -64,6 +64,7 @@ class Collection(Instance):
     Relations are (s, p, o, d=None)-triples with an optional fourth field
     `d`, specifying the datatype of the object.  The datatype may have the
     following values:
+
       - None: object is an IRI.
       - Starts with '@': object is a language-tagged plain literal.
         The language identifier follows the '@'-sign.
@@ -85,7 +86,7 @@ class Collection(Instance):
 
         Arguments:
             src: Storage instance | url | driver
-            location: str
+            location:
                 File path to load from when `src` is a driver.
             options: str
                 Options passed to the storage plugin when `src` is a driver.

bindings/python/dlite-entity-python.i

@@ -404,6 +404,7 @@ def get_instance(
         # Allow metaid to be an Instance
         if isinstance(metaid, Instance):
             metaid = metaid.uri
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             metaid=metaid, dims=dimensions, id=id,
             dimensions=(), properties=()  # arrays must not be None
@@ -459,7 +460,7 @@ def get_instance(
             protocol+driver://location?options#id
             protocol://location?driver=<driver>;options#id
 
-        where `protocol`, `driver`, `location`, `options` and `id are
+        where `protocol`, `driver`, `location`, `options` and `id` are
         documented in the load() method.
 
         If `metaid` is provided, the instance is tried mapped to this
@@ -488,6 +489,7 @@ def get_instance(
                 metaid=metaid
             )
         else:
+            errclr()  # Clear internal error before calling Instance()
             inst = Instance(
                 url=url, metaid=metaid,
                 dims=(), dimensions=(), properties=()  # arrays
@@ -502,6 +504,7 @@ def get_instance(
         If `metaid` is provided, the instance is tried mapped to this
         metadata before it is returned.
         """
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             storage=storage, id=id, metaid=metaid,
             dims=(), dimensions=(), properties=()  # arrays
@@ -519,6 +522,7 @@ def get_instance(
         from dlite.options import make_query
         if options and not isinstance(options, str):
             options = make_query(options)
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             driver=driver, location=str(location), options=options, id=id,
             metaid=metaid,
@@ -529,6 +533,7 @@ def get_instance(
     @classmethod
     def from_json(cls, jsoninput, id=None, metaid=None):
         """Load the instance from json input."""
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             jsoninput=jsoninput, id=id, metaid=metaid,
             dims=(), dimensions=(), properties=()  # arrays
@@ -538,6 +543,7 @@ def get_instance(
     @classmethod
     def from_bson(cls, bsoninput):
         """Load the instance from bson input."""
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             bsoninput=bsoninput,
             dims=(), dimensions=(), properties=()  # arrays
@@ -593,6 +599,7 @@ def get_instance(
         """Create a new metadata entity (instance of entity schema) casted
         to an instance.
         """
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             uri=uri, dimensions=dimensions, properties=properties,
             description=description,
@@ -614,6 +621,7 @@ def get_instance(
             meta = get_instance(metaid)
             dimensions = [dimensions[dim.name]
                           for dim in meta.properties['dimensions']]
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             metaid=metaid, dims=dimensions, id=id,
             dimensions=(), properties=()  # arrays must not be None
@@ -630,10 +638,12 @@ def get_instance(
         warnings.warn(
             "create_from_url() is deprecated, use from_url() instead.",
             DeprecationWarning, stacklevel=2)
-        return Instance(
+        errclr()  # Clear internal error before calling Instance()
+        inst = Instance(
             url=url, metaid=metaid,
             dims=(), dimensions=(), properties=()  # arrays
         )
+        return instance_cast(inst)
 
     @classmethod
     def create_from_storage(cls, storage, id=None, metaid=None):
@@ -646,6 +656,7 @@ def get_instance(
         warnings.warn(
             "create_from_storage() is deprecated, use from_storage() instead.",
             DeprecationWarning, stacklevel=2)
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             storage=storage, id=id, metaid=metaid,
             dims=(), dimensions=(), properties=()  # arrays
@@ -664,12 +675,30 @@ def get_instance(
         from dlite.options import make_query
         if options and not isinstance(options, str):
             options = make_query(options)
+        errclr()  # Clear internal error before calling Instance()
         inst = Instance(
             driver=driver, location=str(location), options=options, id=id,
             dims=(), dimensions=(), properties=()  # arrays
         )
         return instance_cast(inst)
 
+    @classmethod
+    def get_uuids(cls, driver, location, options=None, pattern=None):
+        """Returns a iterator over matching UUIDs in storage.
+
+          Arguments:
+              driver: Name of storage plugin for data parsing.
+              location: Location of resource.  Typically a URL or file path.
+              options: Options passed to the protocol and driver plugins.
+              pattern: A glob pattern matching metadata UUIDs.  If given,
+                  only matching UUIDs will be returned.
+
+          Return:
+              Iterator over all matching UUIDs in storage.
+        """
+        with Storage(driver, location, options=options) as s:
+             return s.get_uuids(pattern=pattern)
+
     def save(self, *dest, location=None, options=None):
         """Saves this instance to url or storage.
 

bindings/python/dlite-misc-python.i

@@ -28,9 +28,9 @@ class errctl():
             shown/hidden.
         filename: Filename to redirect errors to.  The following values
             are handled specially:
-              - "None" or empty: No output is written.
-              - "<stderr>": Write errors to stderr (default).
-              - "<stdout>": Write errors to stdout.
+            - "None" or empty: No output is written.
+            - "<stderr>": Write errors to stderr (default).
+            - "<stdout>": Write errors to stdout.
 
     """
     def __init__(self, hide=(), show=(), filename="<stderr>"):

bindings/python/quantity.py

@@ -1,15 +1,13 @@
-
-""" Define the singleton QuantityHelper to work with pint Quantity and dlite 
-    instance properties.
+"""Define the singleton QuantityHelper to work with pint Quantity and dlite
+instance properties.
 """
 
 from typing import Any
 import numpy as np
-HAS_PINT = True
-try:
-    import pint
-except Exception:
-    HAS_PINT = False
+from dlite.testutils import importcheck
+
+pint = importcheck("pint")
+
 
 DLITE_QUANTITY_TYPES = [
     'int', 'float', 'double', 'uint',
@@ -21,8 +19,16 @@
 
 
 class QuantityHelper:
+    """Singleton class for working with pint Quantity and dlite instance
+    properties.
+    """
 
     def __init__(self):
+        if not pint:
+            raise RuntimeError(
+                'you must install "pint" to work with quantities, '
+                'try: pip install pint'
+            )
         self.__dict__['_instance'] = None
         self.__dict__['_registry'] = None
 
@@ -46,7 +52,7 @@ def _get_unit_as_string(self, unit: Any) -> str:
             unit_string = str(unit)
         return unit_string.replace('%', 'percent')
 
-    def __getitem__(self, name: str) -> pint.Quantity:
+    def __getitem__(self, name: str) -> "pint.Quantity":
         p = self._get_property(name)
         u = self._get_unit_as_string(p.unit)
         return self.quantity(self._instance[name], u)
@@ -105,15 +111,15 @@ def get(self, *names):
         return None
 
     @property
-    def unit_registry(self) -> pint.UnitRegistry:
+    def unit_registry(self) -> "pint.UnitRegistry":
         """ Returns the current pint UnitRegistry object """
         return self._registry.get()
 
-    def quantity(self, magnitude, unit) -> pint.Quantity:
+    def quantity(self, magnitude, unit) -> "pint.Quantity":
         """ Return a pint.Quantity object """
         return self._registry.Quantity(magnitude, unit)
 
-    def parse(self, value: Any) -> pint.Quantity:
+    def parse(self, value: Any) -> "pint.Quantity":
         """ Parse the given value and return a pint.Quantity object """
         if isinstance(value, (pint.Quantity, self._registry.Quantity)):
             return value
@@ -129,8 +135,8 @@ def parse(self, value: Any) -> pint.Quantity:
         else:
             return self.quantity(value, '')
 
-    def parse_expression(self, value: str) -> pint.Quantity:
-        """ Parse an expression (str) and return a pint.Quantity object """
+    def parse_expression(self, value: str) -> "pint.Quantity":
+        """Parse an expression (str) and return a pint.Quantity object """
         result = None
         if value:
             value_str = self._get_unit_as_string(value)
@@ -171,12 +177,6 @@ def to_dict(self, names=None, value_type='quantity', fmt=''):
 
 def get_quantity_helper(instance):
     global quantity_helper
-    if HAS_PINT:
-        if quantity_helper is None:
-            quantity_helper = QuantityHelper()
-        return quantity_helper(instance)
-    else:
-        raise RuntimeError(
-            'you must install "pint" to work with quantities, '
-            'try: pip install pint'
-        )
+    if quantity_helper is None:
+        quantity_helper = QuantityHelper()
+    return quantity_helper(instance)

bindings/python/scripts/CMakeLists.txt

@@ -79,3 +79,13 @@ test_success(
   dlite-validate-empty
   dlite-validate Empty.json
 )
+# Test valid yaml
+test_success(
+  dlite-validate-valid1-yaml
+  dlite-validate ../tests/entities/ValidYaml1.yaml
+)
+# Test invalid yaml
+test_failure(
+  dlite-validate-invalid1-yaml
+  dlite-validate ../tests/entities/InvalidYaml1.yaml
+)

bindings/python/scripts/dlite-validate

@@ -11,7 +11,7 @@ DLITE_BEHAVIOR = True  # Turn off warnings about behavior changes
 import dlite
 
 
-def parse(url, driver=None, options=None, id=None):
+def parse(url, driver=None, options="mode=r", id=None):
     """Loads an instance from storage.
 
     Arguments:

bindings/python/tests/entities/InvalidYaml1.yaml

@@ -0,0 +1,14 @@
+# An invalid version of datamodel Valid1.yaml
+uri: http://onto-ns.com/meta/0.1/InvalidYaml1
+description: An datamodel for an item with float type and invalid shape name.
+dimensions:
+  nf: Number of eigen-frequencies.
+properties:
+  name:
+    type: str
+    description: Name of the item.
+  f:
+    type: float64
+    shape: [nf_MISPELLED]
+    unit: Hz
+    description: The magic eigen-frequencies of the item.

bindings/python/tests/entities/ValidYaml1.yaml

@@ -0,0 +1,14 @@
+# An valid version of datamodel Invalid1.yaml
+uri: http://onto-ns.com/meta/0.1/ValidYaml1
+description: An datamodel for an item with float type and invalid shape name.
+dimensions:
+  nf: Number of eigen-frequencies.
+properties:
+  name:
+    type: str
+    description: Name of the item.
+  f:
+    type: float64
+    shape: [nf]
+    unit: Hz
+    description: The magic eigen-frequencies of the item.

bindings/python/tests/input/test_ref_type_middle.yaml

@@ -0,0 +1,11 @@
+http://onto-ns.com/meta/0.2/Middle:
+  description: Middle-level nested data structure.
+  dimensions: []
+  properties:
+  - name: name
+    type: string
+    description: Value of this structure.
+  - name: leaf
+    type: ref
+    $ref: http://onto-ns.com/meta/0.1/Leaf
+    description: Reference to low-level structure.

bindings/python/tests/test_entity.py

@@ -343,10 +343,37 @@
     assert item.q.f.m.tolist() == [0., 1., 0.001]
     assert item.q.f.to("1/hour").m.tolist() == [0, 3600, 3.6]
 
-    
+
 # For issue #750 - test instance_cast()
 with raises(dlite.DLiteTypeError):
     dlite.instance_cast(inst, dlite.Metadata)
 castinst = dlite.instance_cast(inst.meta, dlite.Instance)
 assert type(castinst) == dlite.Instance
 assert type(dlite.instance_cast(castinst)) == dlite.Metadata
+
+
+# Test storage query
+uuids = {
+    '850637b9-1d21-573c-91b6-477530e4bf58',
+    '020e411b-f349-5689-8657-f82b709369c3',
+    '570611f5-96b3-5b0d-90ad-f3a4c19a78b2',
+    '5e378ac7-83c9-5d77-ab20-b5bb32c695da',
+    'e5efe084-27f2-5fec-9b1c-fa1a692e1434',
+}
+with dlite.Storage("json", indir / "test_ref_type.json") as s:
+    assert set(s.get_uuids()) == uuids
+    assert set(s.get_uuids("http://onto-ns.com/meta/0.3/EntitySchema")) == uuids
+    assert s.get_uuids("xxx") == []
+assert set(
+    dlite.Instance.get_uuids("json", indir / "test_ref_type.json")
+) == uuids
+assert set(
+    dlite.Instance.get_uuids(
+        "json", indir / "test_ref_type.json",
+        pattern="http://onto-ns.com/meta/0.3/EntitySchema",
+    )
+) == uuids
+assert dlite.Instance.get_uuids(
+    "json", indir / "test_ref_type.json",
+    pattern="xxx",
+) == []

bindings/python/tests/test_ref_type.py

@@ -1,15 +1,24 @@
 from pathlib import Path
 
 import dlite
+from dlite.testutils import importcheck
+
+yaml = importcheck("yaml")
 
 
 thisdir = Path(__file__).resolve().parent
 indir = thisdir / "input"
 
-dlite.storage_path.append(indir / "test_ref_type.json")
+dlite.storage_path.append(indir)
+dlite.storage_path.append(indir / "test_ref_type_middle.yaml")
+
+# If yaml is available, we read Middle v0.2, which is defined in
+# `test_ref_type_middle.yaml`.  Otherwise, we read Middle v0.1, which
+# is defined together with the other datamodels in `test_ref_type.json`.
+version = "0.2" if yaml else "0.1"
 
 Top = dlite.get_instance("http://onto-ns.com/meta/0.1/Top")
-Middle = dlite.get_instance("http://onto-ns.com/meta/0.1/Middle")
+Middle = dlite.get_instance(f"http://onto-ns.com/meta/{version}/Middle")
 Leaf = dlite.get_instance("http://onto-ns.com/meta/0.1/Leaf")
 Linked = dlite.get_instance("http://onto-ns.com/meta/0.1/Linked")
 Tree = dlite.get_instance("http://onto-ns.com/meta/0.1/Tree")
@@ -78,6 +87,8 @@
 assert cyclic.subtree[0].subtree[0] == cyclic
 assert cyclic.subtree[0].subtree[0].subtree[0] == cyclic
 
-# Instantiate nested from dict
-# For issue #515
-# middle = Middle(properties={"name": "nested", "leaf": {"a": 1, "b": True}})
+# For isue #982: ref-type in yaml
+assert Middle.getprop("leaf").ref == "http://onto-ns.com/meta/0.1/Leaf"
+
+# For issue #515: Instantiate nested from dict
+#middle = Middle(properties={"name": "nested", "leaf": {"a": 1, "b": True}})