diff --git a/.gitignore b/.gitignore index eda7028eb..0fad5f096 100644 --- a/.gitignore +++ b/.gitignore @@ -54,9 +54,11 @@ pip-wheel-metadata .project .pydevproject .settings +coverage.xml # Mac OSX .DS_Store # ipython .ipynb_checkpoints +/.pytest_cache/ diff --git a/CHANGES.rst b/CHANGES.rst index 643e5c6dd..7aa3ec1fe 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,10 @@ Enhancements and Fixes - registry.Ivoid now accepts multiple ivoids and will then match any of them. [#517] +- Introducing the new MIVOT module, enabling processed VOTable data mapped to + any model serialized in VO-DML. This package dynamically generates python objects + whose structure corresponds to the classes of the mapped models. [#497] + Deprecations and Removals ------------------------- diff --git a/conftest.py b/conftest.py index 7c7612944..53d14fbd0 100644 --- a/conftest.py +++ b/conftest.py @@ -54,6 +54,7 @@ def pytest_configure(config): # packages for which version numbers are displayed when running the tests. PYTEST_HEADER_MODULES['Astropy'] = 'astropy' # noqa PYTEST_HEADER_MODULES['requests'] = 'requests' # noqa + PYTEST_HEADER_MODULES['defusedxml'] = 'defusedxml' PYTEST_HEADER_MODULES.pop('Pandas', None) PYTEST_HEADER_MODULES.pop('h5py', None) diff --git a/docs/index.rst b/docs/index.rst index 093354483..d4485e5de 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -134,5 +134,6 @@ Using ``pyvo`` registry/index io/index auth/index + mivot/index utils/index utils/prototypes diff --git a/docs/mivot/_images/mangoEpochPosition.png b/docs/mivot/_images/mangoEpochPosition.png new file mode 100644 index 000000000..3d158c736 Binary files /dev/null and b/docs/mivot/_images/mangoEpochPosition.png differ diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst new file mode 100644 index 000000000..83be5f839 --- /dev/null +++ b/docs/mivot/index.rst @@ -0,0 +1,235 @@ +******************** +MIVOT (`pyvo.mivot`) +******************** + +This module contains the new feature of annotations in VOTable. +Astropy version >= 6.0 is required. + +Introduction +============ +.. pull-quote:: + + Model Instances in VOTables (MIVOT) defines a syntax to map VOTable + data to any model serialized in VO-DML. The annotation operates as a + bridge between the data and the model. It associates the column/param + metadata from the VOTable to the data model elements (class, attributes, + types, etc.) [...]. + The data model elements are grouped in an independent annotation block + complying with the MIVOT XML syntax. This annotation block is added + as an extra resource element at the top of the VOTable result resource. The + MIVOT syntax allows to describe a data structure as a hierarchy of classes. + It is also able to represent relations and composition between them. It can + also build up data model objects by aggregating instances from different + tables of the VOTable. + +- Model Instances in VOTables is a VO `standard `_ +- Requires Astropy>=6.0 +- ``pyvo.mivot`` is a prototype feature which must be activated with ``activate_features("MIVOT")`` + + +Implementation Scope +-------------------- +This implementation is totally model-agnostic. + +- It does not operate any validation against specific data models. +- It just requires the annotation syntax being compliant with the standards. + +However, many data samples used for the test suite and provided as examples +are based on the ``EpochPropagation`` class of the ``Mango`` data model +that is still a draft. +This class collects all the parameters we need to compute the epoch propagation of moving sky objects. +Some of the examples have been provided by a special end-point of the Vizier cone-search service +(https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch) that maps query results to this model. + +.. image:: _images/mangoEpochPosition.png + :width: 500 + :alt: EpochPropagation class used to validate this api. + +It is to be noted that the Vizier service does not annotate errors at the time of writing (Q1 2024) + +The implementation uses the Astropy read/write annotation module (6.0+), +which allows to get (and set) Mivot blocks from/into VOTables as an XML element serialized as a string. + +.. pull-quote:: + + Not all MIVOT features are supported by this implementation, which mainly focuses on the + epoch propagation use case: + + - ``JOIN`` features are not supported. + - ``TEMPLATES`` with more than one ``INSANCE`` not supported. + +Integrated Readout +------------------ +The ``ModelViewer`` module manages access to data mapped to a model through dynamically +generated objects (``MivotInstance``class). +The example below shows how a VOTable, resulting from a cone-search query which data are mapped +to the ``EpochPosition`` class, can be consumed. + +.. doctest-remote-data:: + >>> import astropy.units as u + >>> from astropy.coordinates import SkyCoord + >>> from pyvo.dal.scs import SCSService + >>> from pyvo.utils.prototype import activate_features + >>> from pyvo.mivot.version_checker import check_astropy_version + >>> from pyvo.mivot.viewer.mivot_viewer import MivotViewer + >>> activate_features("MIVOT") + >>> if check_astropy_version() is False: + ... pytest.skip("MIVOT test skipped because of the astropy version.") + >>> scs_srv = SCSService("https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch/I/239/hip_main") + >>> m_viewer = MivotViewer( + ... scs_srv.search( + ... pos=SkyCoord(ra=52.26708 * u.degree, dec=59.94027 * u.degree, frame='icrs'), + ... radius=0.05 + ... ) + ... ) + >>> mivot_instance = m_viewer.dm_instance + >>> print(mivot_instance.dmtype) + EpochPosition + >>> print(mivot_instance.Coordinate_coordSys.spaceRefFrame.value) + ICRS + >>> while m_viewer.next(): + ... print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}") + position: 59.94033461 52.26722684 + + +In this example, the data readout is totally managed by the ``MivotViewer`` instance. +The ``astropy.io.votable`` API is encapsulated in this module. + +Model leaves (class attributes) are complex types that provide additional information: + +- ``value``: attribute value +- ``dmtype``: attribute type such as defined in the Mivot annotations +- ``unit``: attribute unit such as defined in the Mivot annotations +- ``ref``: identifier of the table column mapped on the attribute + +The model view on a data row can also be passed as a Python dictionary +using the ``dict`` property of ``MivotInstance``. + +.. code-block:: python + :caption: Working with a model view as a dictionary + (the JSON layout has been squashed for display purpose) + + from pyvo.mivot.utils.dict_utils import DictUtils + + mivot_instance = m_viewer.dm_instance + mivot_object_dict = mivot_object.dict + + DictUtils.print_pretty_json(mivot_object_dict) + { + "dmtype": "EpochPosition", + "longitude": {"value": 359.94372764, "unit": "deg"}, + "latitude": {"value": -0.28005255, "unit": "deg"}, + "pmLongitude": {"value": -5.14, "unit": "mas/yr"}, + "pmLatitude": {"value": -25.43, "unit": "mas/yr"}, + "epoch": {"value": 1991.25, "unit": "year"}, + "Coordinate_coordSys": { + "dmtype": "SpaceSys", + "dmid": "SpaceFrame_ICRS", + "dmrole": "coordSys", + "spaceRefFrame": {"value": "ICRS"}, + }, + } + +- It is recommended to use a copy of the + dictionary as it will be rebuilt each time the ``dict`` property is invoked. +- The default representation of ``MivotInstance`` instances is made with a pretty + string serialization of this dictionary. + +Per-Row Readout +--------------- + +The annotation schema can also be applied to table rows read outside of the ``MivotViewer`` +with the `astropy.io.votable` API: + +.. code-block:: python + :caption: Accessing the model view of Astropy table rows + + votable = parse(path_to_votable) + table = votable.resources[0].tables[0] + # init the viewer + mivot_viewer = MivotViewer(votable, resource_number=0) + mivot_object = mivot_viewer.dm_instance + # and feed it with the table row + read = [] + for rec in table.array: + mivot_object.update(rec) + read.append(mivot_object.longitude.value) + # show that the model retrieve the correct data values + assert rec["RAICRS"] == mivot_object.longitude.value + assert rec["DEICRS"] == mivot_object.latitude.value + +In this case, it is up to the user to ensure that the read data rows are those mapped by the Mivot annotations. + +For XML Hackers +--------------- + +The model instances can also be serialized as XML elements that can be parsed with XPath queries. + +.. code-block:: python + :caption: Accessing the XML view of the mapped model instances + + with MivotViewer(path_to_votable) as mivot_viewer: + while mivot_viewer.next(): + xml_view = mivot_viewer.xml_view + # do whatever you want with this XML element + +It to be noted that ``mivot_viewer.xml_view`` is a shortcut +for ``mivot_viewer.xml_view.view`` where ``mivot_viewer.xml_view`` +is is an instance of ``pyvo.mivot.viewer.XmlViewer``. +This object provides many functions facilitating the XML parsing. + +Class Generation in a Nutshell +------------------------------ + +MIVOT reconstructs model structures with 3 elements: + +- ``INSTANCE`` for the objects +- ``ATTRIBUTE`` for the attributes +- ``COLLECTION`` for the elements with a cardinality greater than 1 + +The role played by each of these elements in the model hierarchy is defined +by its ``@dmrole`` XML attribute. Types of both ``INSTANCE`` and ``ATTRIBUTE`` are defined by +their ``@dmtype`` XML attributes. + +``MivotInstance`` classes are built by following MIVOT annotation structure: + +- ``INSTANCE`` are represented by Python classes +- ``ATTRIBUTE`` are represented by Python class fields +- ``COLLECTION`` are represented by Python lists ([]) + +``@dmrole`` and ``@dmtype`` cannot be used as Python keywords as such, because they are built from VO-DML +identifiers, which have the following structure: ``model:a.b``. + +- Only the last part of the path is kept for attribute names. +- For class names, forbidden characters (``:`` or ``.``) are replaced with ``_``. +- Original ``@dmtype`` are kept as attributes of generated Python objects. +- The structure of the ``MivotInstance`` objects can be inferred from the mapped model in 2 different ways: + + - 1. From the MIVOT instance property ``MivotInstance.dict`` a shown above. + This is a pure Python dictionary but its access can be slow because it is generated + on the fly each time the property is invoked. + - 2. From the internal class dictionary ``MivotInstance.__dict__`` + (see the Python `data model `_). + + .. code-block:: python + :caption: Exploring the MivotInstance structure with the internal dictionaries + + mivot_instance = mivot_viewer.dm_instance + + print(mivot_instance.__dict__.keys()) + dict_keys(['dmtype', 'longitude', 'latitude', 'pmLongitude', 'pmLatitude', 'epoch', 'Coordinate_coordSys']) + + print(mivot_instance.Coordinate_coordSys.__dict__.keys()) + dict_keys(['dmtype', 'dmid', 'dmrole', 'spaceRefFrame']) + + print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys()) + dict_keys(['dmtype', 'value', 'unit', 'ref']) + +Reference/API +============= + +.. automodapi:: pyvo.mivot +.. automodapi:: pyvo.mivot.viewer +.. automodapi:: pyvo.mivot.seekers +.. automodapi:: pyvo.mivot.features +.. automodapi:: pyvo.mivot.utils diff --git a/pyvo/conftest.py b/pyvo/conftest.py index ffc02f2ab..48464d3d8 100644 --- a/pyvo/conftest.py +++ b/pyvo/conftest.py @@ -42,6 +42,7 @@ def pytest_configure(config): PYTEST_HEADER_MODULES.pop('h5py', None) PYTEST_HEADER_MODULES.pop('Scipy', None) PYTEST_HEADER_MODULES.pop('Matplotlib', None) + PYTEST_HEADER_MODULES['defusedxml'] = 'defusedxml' from . import __version__ TESTED_VERSIONS['pyvo'] = __version__ diff --git a/pyvo/mivot/__init__.py b/pyvo/mivot/__init__.py new file mode 100644 index 000000000..c096ba11e --- /dev/null +++ b/pyvo/mivot/__init__.py @@ -0,0 +1,2 @@ +# package entry point +from .viewer.mivot_viewer import MivotViewer \ No newline at end of file diff --git a/pyvo/mivot/features/__init__.py b/pyvo/mivot/features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyvo/mivot/features/static_reference_resolver.py b/pyvo/mivot/features/static_reference_resolver.py new file mode 100644 index 000000000..8e194c5c8 --- /dev/null +++ b/pyvo/mivot/features/static_reference_resolver.py @@ -0,0 +1,71 @@ +""" +Class used to resolve each static REFERENCE found in mivot_block. +""" +from copy import deepcopy +from pyvo.mivot.utils.exceptions import MivotException +from pyvo.mivot.utils.xpath_utils import XPath +from pyvo.utils.prototype import prototype_feature + + +@prototype_feature('MIVOT') +class StaticReferenceResolver: + """ + Namespace for the function processing the static REFERENCEs + """ + @staticmethod + def resolve(annotation_seeker, templates_ref, mivot_block): + """ + Resolve all static REFERENCEs found in the mivot_block. + The referenced objects are first searched in GLOBALS and then in the templates_ref table. + REFERENCE elements are replaced with the referenced objects set with the roles of the REFERENCEs. + Works even if REFERENCE tags are numbered by the former processing. + Parameters + ---------- + annotation_seeker : AnnotationSeeker + Utility to extract desired elements from the mapping block. + templates_ref : str + Identifier of the table where the mivot_block comes from. + mivot_block : xml.etree.ElementTree + The XML element object. + Returns + ------- + int + The number of references resolved. + Raises + ------ + MappingException + If the reference cannot be resolved. + NotImplementedError + If the reference is dynamic. + """ + resolved_refs = 0 + for ele in XPath.x_path_startwith(mivot_block, './/REFERENCE_'): + dmref = ele.get("dmref") + # If we have no @dmref in REFERENCE, we consider this is a ref based on a keys + if dmref is None: + raise NotImplementedError("Dynamic reference not implemented") + target = annotation_seeker.get_globals_instance_by_dmid(dmref) + found_in_global = True + if target is None and templates_ref is not None: + target = annotation_seeker.get_templates_instance_by_dmid(templates_ref, dmref) + found_in_global = False + if target is None: + raise MivotException(f"Cannot resolve reference={dmref}") + # Resolve static references recursively + if not found_in_global: + StaticReferenceResolver.resolve(annotation_seeker, templates_ref, ele) + else: + StaticReferenceResolver.resolve(annotation_seeker, None, ele) + # Set the reference role to the copied instance + target_copy = deepcopy(target) + # If the reference is within a collection: no role + if ele.get('dmrole'): + target_copy.attrib["dmrole"] = ele.get('dmrole') + parent_map = {c: p for p in mivot_block.iter() for c in p} + parent = parent_map[ele] + # Insert the referenced object + parent.append(target_copy) + # Drop the reference + parent.remove(ele) + resolved_refs += 1 + return resolved_refs diff --git a/pyvo/mivot/seekers/__init__.py b/pyvo/mivot/seekers/__init__.py new file mode 100644 index 000000000..9e7047270 --- /dev/null +++ b/pyvo/mivot/seekers/__init__.py @@ -0,0 +1,4 @@ +""" +``seekers`` package contains utilities for retrieving +components of VOTales or of Mivot blocks +""" \ No newline at end of file diff --git a/pyvo/mivot/seekers/annotation_seeker.py b/pyvo/mivot/seekers/annotation_seeker.py new file mode 100644 index 000000000..4f74eca5c --- /dev/null +++ b/pyvo/mivot/seekers/annotation_seeker.py @@ -0,0 +1,415 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Utilities for extracting sub-blocks from a MIVOT mapping block. +""" +import logging +from pyvo.mivot.utils.exceptions import MivotException, MappingException +from pyvo.mivot.utils.vocabulary import Att, Ele +from pyvo.mivot.utils.vocabulary import Constant +from pyvo.mivot.utils.xpath_utils import XPath +from pyvo.utils.prototype import prototype_feature + + +@prototype_feature('MIVOT') +class AnnotationSeeker: + """ + This class provides tools for extracting mapping sub-blocks commonly used by other stake-holders. + All functions using the mapping employ this class to obtain XML elements. + To simplify the job for other tools, the XML namespace is removed from the mapping block. + Attributes: + ---------- + _xml_block (~`xml.etree.ElementTree.Element`): Full mapping block. + _globals_block (~`xml.etree.ElementTree.Element` or None): GLOBALS block. + _templates_blocks (dict): Templates dictionary where keys are tableref and values are XML-TEMPLATES. + """ + def __init__(self, xml_block): + """ + Initializes the AnnotationSeeker. + - Split the mapping as elements of interest. + - Remove the name_spaces. + - Append numbers to JOIN/REFERENCE. + Parameters + ---------- + xml_block (~`xml.etree.ElementTree.Element`): XML mapping block + """ + self._xml_block = xml_block + self._globals_block = None + self._templates_blocks = {} + self._find_globals_block() + self._find_templates_blocks() + self._rename_ref_and_join() + + def _find_globals_block(self): + """ + Extract the GLOBALS element from the XML mapping block + and store its reference. + """ + for child in self._xml_block: + if self._name_match(child.tag, Ele.GLOBALS): + logging.debug("Found " + Ele.GLOBALS) + self._globals_block = child + + def _find_templates_blocks(self): + """ + Search the TEMPLATES elements from the XML mapping block and store its reference. + This method iterates through the children of the XML mapping block, identifies TEMPLATES blocks, + and associates them with their respective @tableref values in the _templates_blocks dictionary. + Returns + ------- + dict: TEMPLATES tablerefs and their mapping blocks {'tableref': mapping_block, ...} + Raises + ------ + MivotElementNotFound + If a TEMPLATES block is found without a @tableref attribute, + and there are already other TEMPLATES blocks. + """ + for child in self._xml_block: + if self._name_match(child.tag, Ele.TEMPLATES): + tableref = child.get(Att.tableref) + if tableref is not None: + logging.debug("Found " + Ele.TEMPLATES + " %s", tableref) + self._templates_blocks[tableref] = child + elif not self._templates_blocks: + logging.debug("Found " + Ele.TEMPLATES + " without " + Att.tableref) + self._templates_blocks["DEFAULT"] = child + else: + raise MivotException(Ele.TEMPLATES + " without " + Att.tableref + " must be unique") + + def _rename_ref_and_join(self): + """ + Tag specified elements with a numerical suffix to make them unique. + This is necessary to avid confusions when resolving cross references + between elements + The elements that are renamed are: + - JOIN + - REFERENCE + """ + cpt = 1 + for tag in ['REFERENCE', 'JOIN']: + xpath = './/' + tag + for ele in XPath.x_path(self._xml_block, xpath): + ele.tag = tag + '_' + str(cpt) + cpt += 1 + + @staticmethod + def _name_match(name, expected): + """ + Return true if name matches expected whatever the namespace + Parameters + ---------- + name (str): Name to compare + expected (str): Expected name for the comparison + Returns + ------- + bool + """ + if type(name).__name__ == 'cython_function_or_method': + return False + return name.endswith(expected) + + """ + Properties + """ + @property + def globals_block(self): + """ + GLOBALS getter + """ + return self._globals_block + + def get_globals_collections(self): + """ + Return a list of all GLOBALS/COLLECTION elements. + These collections have no @dmroles but often @dmids. + They have particular roles + - Used by references (e.g., filter definition) + - Used as head of the mapped model (e.g., [Cube instance]) + Returns + ------- + list: GLOBALS/COLLECTION elements + """ + return XPath.x_path(self._globals_block, ".//COLLECTION") + + def get_models(self): + """ + Get the declared MODELs and their URLs. + Returns + ------- + dict: MODELs and their URLs {'model': [url], ...} + """ + models_found = {} + eset = XPath.x_path(self._xml_block, ".//" + Ele.MODEL) + for ele in eset: + models_found[ele.get("name")] = ele.get("url") + return models_found + + def get_templates_tableref(self): + """ + Get all @tableref of the mapping. + Returns + ------- + list: @tableref found in the mapping + """ + return self._templates_blocks.keys() + + def get_templates(self): + """ + Return a list of TEMPLATES @tableref. + Returns + ------- + list: TEMPLATES tablerefs + """ + templates_found = [] + eset = XPath.x_path(self._xml_block, ".//" + Ele.TEMPLATES) + for ele in eset: + tableref = ele.get("tableref") + if tableref is None: + tableref = Constant.FIRST_TABLE + templates_found.append(tableref) + return templates_found + + def get_templates_block(self, tableref): + """ + Return the TEMPLATES mapping block of the table matching @tableref. + If tableref is None returns all values of templates_blocks. + Parameters + ---------- + tableref (str): @tableref of the searched TEMPLATES + Returns + ------- + dict: TEMPLATES tablerefs and their mapping blocks {'tableref': mapping_block, ...} + """ + # one table: name forced to DEFAULT or take the first + if tableref is None or tableref == Constant.FIRST_TABLE: + for _, tmpl in self._templates_blocks.items(): + return tmpl + return self._templates_blocks[tableref] + + """ + INSTANCE + """ + def get_instance_dmtypes(self): + """ + Get @dmtypes of all mapped instances + Returns + ------- + dict: @dmtypes of all mapped instances {GLOBALS: [], TEMPLATES: {}} + """ + dmtypes_found = {Ele.GLOBALS: [], Ele.TEMPLATES: {}} + eset = XPath.x_path(self._globals_block, ".//" + Ele.INSTANCE) + for ele in eset: + dmtypes_found[Ele.GLOBALS].append(ele.get(Att.dmtype)) + for tableref, block in self._templates_blocks.items(): + dmtypes_found[Ele.TEMPLATES][tableref] = [] + eset = XPath.x_path(block, ".//" + Ele.INSTANCE) + for ele in eset: + dmtypes_found[Ele.TEMPLATES][tableref].append(ele.get(Att.dmtype)) + return dmtypes_found + + def get_instance_by_dmtype(self, dmtype_pattern): + """ + Get all the mapped instances that have a @dmtype containing dmtype_pattern + Parameters + ---------- + dmtype_pattern (str): @dmtype looked for + Returns + ------- + dict: @dmtypes of all mapped instances {'dmtype': [instance], ...} + """ + instance_found = {Ele.GLOBALS: [], Ele.TEMPLATES: {}} + eset = XPath.x_path_contains(self._globals_block, + ".//" + Ele.INSTANCE, + Att.dmtype, + dmtype_pattern + ) + instance_found[Ele.GLOBALS] = eset + for (tableref, block) in self._templates_blocks.items(): + instance_found[Ele.TEMPLATES][tableref] = XPath.x_path_contains(block, './/' + + Ele.INSTANCE, Att.dmtype, dmtype_pattern) + return instance_found + + """ + GLOBALS INSTANCES + """ + def get_globals_instances(self): + """ + Return the list of all GLOBALS/INSTANCE elements. + These collections have no @dmroles but often @dmids. + They have particular roles when: + - Used by references (e.g., filter definition) + - Used as head of the mapped model (e.g., Cube instance) + Returns + ------- + list: GLOBALS/INSTANCE elements + """ + return XPath.x_path(self._globals_block, "./" + Ele.INSTANCE) + + def get_globals_instance_dmids(self): + """ + Get a list of @dmid for GLOBALS/INSTANCE. + Returns + ------- + list: @dmid of GLOBALS/INSTANCE + """ + dmids_found = [] + eset = XPath.x_path(self._globals_block, + ".//" + Ele.INSTANCE + "[@" + Att.dmid + "]") + for ele in eset: + dmids_found.append(ele.get(Att.dmid)) + return dmids_found + + def get_globals_instance_by_dmid(self, dmid): + """ + Get the GLOBALS/INSTANCE with @dmid=dmid. + Parameters + ---------- + dmid (str): @dmid of the searched GLOBALS/INSTANCE + Returns + ------- + dict: `~xml.etree.ElementTree.Element` + """ + eset = XPath.x_path(self._globals_block, + (".//" + Ele.INSTANCE + "[@" + Att.dmid + "='" + dmid + "']") + ) + for ele in eset: + return ele + return None + + def get_globals_instance_dmtypes(self): + """ + Get the list the @dmtype GLOBALS/INSTANCE. + Returns + ------- + list: @dmtype of GLOBALS/INSTANCE + """ + dmtypes_found = [] + for inst in self.get_globals_instances(): + dmtypes_found.append(inst.get(Att.dmtype)) + return dmtypes_found + + def get_templates_instance_by_dmid(self, tableref, dmid): + """ + Get the TEMPLATES/INSTANCE with @dmid=dmid and TEMPLATES@tableref=tableref. + Parameters + ---------- + tableref (str): @tableref of the searched TEMPLATES + dmid (str): @dmid of the searched TEMPLATES/INSTANCE + Returns + ------- + dict: ~`xml.etree.ElementTree.Element` + """ + templates_block = self.get_templates_block(tableref) + if templates_block is None: + return None + eset = XPath.x_path(templates_block, + ".//" + Ele.INSTANCE + "[@" + Att.dmid + "='" + dmid + "']") + for ele in eset: + return ele + return None + + def get_globals_instance_from_collection(self, sourceref, pk_value): + """ + Get the GLOBALS/COLLECTION[@dmid=sourceref]/INSTANCE/PRIMARY_KEY[@value='pk_value']. + Parameters + ---------- + sourceref (str): @dmid of the searched GLOBALS/COLLECTION + pk_value: (str): @value of the searched GLOBALS/COLLECTION/INSTANCE/PRIMARY_KEY + Returns + ------- + dict: ~`xml.etree.ElementTree.Element` + """ + einst = XPath.x_path( + self._globals_block, ".//" + Ele.COLLECTION + "[@" + Att.dmid + "='" + + sourceref + "']/" + Ele.INSTANCE + "/" + Att.primarykey + + "[@" + Att.value + "='" + pk_value + "']") + parent_map = {c: p for p in self._globals_block.iter() for c in p} + for inst in einst: + return parent_map[inst] + return None + + """ + GLOBALS COLLECTION + """ + def get_globals_collection(self, dmid): + """ + Get the GLOBALS/COLLECTION with @dmid=dmid. + Parameters + ---------- + dmid (str):@dmid of the searched GLOBALS/COLLECTION + Returns + ------- + dict: `xml.etree.ElementTree.Element` + """ + eset = XPath.x_path(self._globals_block, ".//" + Ele.GLOBALS + "/" + Ele.COLLECTION + + "[@" + Att.dmid + "='" + dmid + "']") + for ele in eset: + return ele + return None + + def get_globals_collection_dmids(self): + """ + Get the list of all the @dmid of GLOBALS/COLLECTION. + Returns + ------- + list: @dmid of GLOBALS/COLLECTION + """ + dmids_found = [] + eset = XPath.x_path(self._globals_block, ".//" + Ele.COLLECTION + "[@" + Att.dmid + "]") + for ele in eset: + dmids_found.append(ele.get(Att.dmid)) + return dmids_found + + def get_globals_collection_dmtypes(self): + """ + Get the list of the @dmtype of GLOBALS/COLLECTION/INSTANCE. + Used for collections of static objects. + Returns + ------- + list: @dmtype of GLOBALS/COLLECTION/INSTANCE + """ + eles = XPath.x_path(self._globals_block, ".//" + Ele.COLLECTION + "/" + Ele.INSTANCE) + dmtypes_found = [] + for inst in eles: + dmtype = inst.get(Att.dmtype) + if dmtype not in dmtypes_found: + dmtypes_found.append(dmtype) + return dmtypes_found + + def get_collection_item_by_primarykey(self, coll_dmid, key_value): + """ + Get the GLOBALS/COLLECTION/INSTANCE with COLLECTION@dmid=dmid and + INSTANCE with a PRIMARY_key which @value matches key_value. + An exception is raised if there is less or more than one element matching the criteria. + The 2 parameters match the dynamic REFERENCE definition. + Parameters + ---------- + coll_dmid (str): @dmid of the searched GLOBALS/COLLECTION + key_value (str): @value of the searched GLOBALS/COLLECTION/INSTANCE/PRIMARY_KEY + Returns + ------- + dict: ~`xml.etree.ElementTree.Element` + Raises + ------ + MivotElementNotFound: If no element matches the criteria. + MappingException: If more than one element matches the criteria. + """ + eset = XPath.x_path(self._globals_block, ".//" + Ele.COLLECTION + "[@" + Att.dmid + "='" + + coll_dmid + "']/" + Ele.INSTANCE + "/" + Att.primarykey + + "[@" + Att.value + "='" + key_value + "']") + if len(eset) == 0: + message = (f"{Ele.INSTANCE} with {Att.primarykey} = {key_value} in " + f"{Ele.COLLECTION} {Att.dmid} {key_value} not found" + ) + raise MivotException(message) + if len(eset) > 1: + message = ( + f"More than one {Ele.INSTANCE} with {Att.primarykey}" + f" = {key_value} found in {Ele.COLLECTION} " + f"{Att.dmid} {key_value}" + ) + raise MappingException(message) + logging.debug(Ele.INSTANCE + " with " + Att.primarykey + "=%s found in " + + Ele.COLLECTION + " " + + Att.dmid + "=%s", key_value, coll_dmid) + parent_map = {c: p for p in self._globals_block.iter() for c in p} + return parent_map[eset[0]] diff --git a/pyvo/mivot/seekers/resource_seeker.py b/pyvo/mivot/seekers/resource_seeker.py new file mode 100644 index 000000000..30067dc40 --- /dev/null +++ b/pyvo/mivot/seekers/resource_seeker.py @@ -0,0 +1,118 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Class that provides multiple getters on VOTable RESOURCE elements. +""" +from pyvo.mivot.utils.vocabulary import Constant +from pyvo.utils.prototype import prototype_feature + + +@prototype_feature('MIVOT') +class ResourceSeeker: + """ + This class provides multiple getters on resource tables. + Some methods are simple wrappers for external tools in order to have all + the search functions on RESOURCE gathered in within a single namespace. + """ + def __init__(self, resource): + """ + Constructor + Parameters + ---------- + resource (astropy.votable.Resource): The resource object to be queried. + """ + self._resource = resource + + def get_table_ids(self): + """ + Return the list of table ids. + Only resource children are considered. + The @ID is first searched and then the @name, and finally 'AnonymousTable' is taken. + Returns + ------- + list of str: table ids. + """ + ids_found = [] + for table in self._resource.tables: + if table.ID is not None: + ids_found.append(table.ID) + elif table.name is not None: + ids_found.append(table.name) + else: + ids_found.append(Constant.ANONYMOUS_TABLE) + return ids_found + + def get_table(self, table_name_or_id): + """ + Return the table matching table_name first by ID and then by name. + Parameters + ---------- + table_name_or_id (str): Name or id of the table to get. + Returns + ------- + ~astropy.votable.table: table matching the table_name. + """ + if table_name_or_id == Constant.FIRST_TABLE: + return self._resource.tables[0] + for table in self._resource.tables: + if (table_name_or_id is None or table.name == table_name_or_id + or table.ID == table_name_or_id): + return table + return None + + def get_params(self): + """ + Return the VOTable PARAMS. + Returns + ------- + ~astropy.votable.Resource.params: VOTable PARAMS. + """ + return self._resource.params + + def get_id_index_mapping(self, table_name): + """ + Build an index binding column number with field id. + Parameters + ---------- + table_name (str): Name of the table. + Returns + ------- + dict: dictionary mapping field id to column number: {name: {ID, ref, indx}...} + """ + column_index = {} + table = self.get_table(table_name) + indx = 0 + for field in table.fields: + field_desc = {} + if field.ID is not None: + field_desc["ID"] = field.ID + if field.ref is not None: + field_desc["ref"] = field.ref + field_desc["indx"] = indx + if "ID" not in field_desc: + field_desc["ID"] = field.name + column_index[field.name] = field_desc + indx += 1 + + return column_index + + def get_id_unit_mapping(self, table_name): + """ + Build an index binding field unit with field id. + Parameters + ---------- + table_name (str): Name of the table. + Returns + ------- + dict: A dictionary mapping field id to field unit {ID1: unit, name1: unit, ref1: unit ...}. + """ + unit_index = {} + table = self.get_table(table_name) + for field in table.fields: + unit = field.unit + if field.ID is not None: + unit_index[field.ID] = unit + elif field.name is not None: + unit_index[field.name] = unit + elif field.ref is not None: + unit_index[field.ref] = unit + return unit_index diff --git a/pyvo/mivot/seekers/table_iterator.py b/pyvo/mivot/seekers/table_iterator.py new file mode 100644 index 000000000..48bc9e1b6 --- /dev/null +++ b/pyvo/mivot/seekers/table_iterator.py @@ -0,0 +1,54 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Iterator for table rows. +""" +from pyvo.utils.prototype import prototype_feature + + +@prototype_feature('MIVOT') +class TableIterator: + """ + Simple wrapper iterating over table rows. + Some methods are simple wrappers for external tools in order to have all + the search functions on TABLE gathered in within a single namespace. + """ + def __init__(self, name, data_table): + """ + Constructor of the TableIterator class. + Parameters + ---------- + name (str): Table name (not really used). + data_table (~numpy.ndarray): Numpy table returned by `~astropy.votable`. + """ + self.name = name + self.data_table = data_table + self.last_row = None + self.iter = None + # not used yet + self.row_filter = None + + def get_next_row(self): + """ + Return the next Numpy row or None. + The end of table exception usually returned by Numpy is trapped. + """ + # The iterator is set at the first iteration + if self.iter is None: + self.iter = iter(self.data_table) + try: + while True: + row = next(self.iter) + if row is not None: + if self.row_filter is None or self.row_filter.row_match(row): + self.last_row = row + return row + else: + return None + except StopIteration: + return None + + def rewind(self): + """ + Set the pointer on the table-top, destroys the iterator actually. + """ + self.iter = None diff --git a/pyvo/mivot/tests/__init__.py b/pyvo/mivot/tests/__init__.py new file mode 100644 index 000000000..eb1d89d15 --- /dev/null +++ b/pyvo/mivot/tests/__init__.py @@ -0,0 +1,127 @@ +""" +Utility class to check that generated XML match reference elements. +Only used by the tests +""" +from pyvo.utils import activate_features + +# Activate MIVOT for all tests +activate_features('MIVOT') + +try: + from defusedxml import ElementTree as etree +except ImportError: + from xml.etree import ElementTree as etree +from pyvo.mivot.utils.xml_utils import XmlUtils + +class XMLOutputChecker: + """ + This class is used to compare XML outputs, ignoring whitespace differences. + """ + def check_output(self, want, got): + """ + Compare two XML outputs, ignoring whitespace differences. + Parameters + ---------- + want : str + The expected XML output. + got : str + The actual XML output. + Returns + ------- + bool + True if the two XML outputs are equal, False otherwise. + """ + return self._format_xml(want.strip()) == self._format_xml(got.strip()) + + def output_difference(self, want, got): + """ + Return a string describing the differences between two XML outputs. + Parameters + ---------- + want : str + The expected XML output. + got : str + The actual XML output. + Returns + ------- + str + A string describing the differences between the two XML outputs. + """ + return f"Diff:\n{self._format_xml(want)}\nvs.\n{self._format_xml(got)}" + + def _format_xml(self, xml_str): + """ + Format an XML string. + Parameters + ---------- + xml_str : str + The XML string to format. + Returns + ------- + str + The formatted XML string. + """ + return "\n".join(line.strip() for line in xml_str.splitlines()) + + @staticmethod + def xmltree_to_file(xmltree, file_path): + """ + Write an XML tree to a file. + Parameters + ---------- + xmltree : ~`xml.etree.ElementTree.Element` + The XML tree to write to the file. + file_path : str + The path to the output file. + """ + with open(file_path, 'w') as output: + output.write(XmlUtils.pretty_string(xmltree)) + + + @staticmethod + def xmltree_from_file(file_path): + """ + Parse an XML tree from a file. + Parameters + ---------- + file_path : str + The path to the XML file. + Returns + ------- + ~`xml.etree.ElementTree.Element` + The parsed XML tree. + """ + return etree.parse(file_path) + + @staticmethod + def assertXmltreeEquals(xmltree1, xmltree2): + """ + Assert that two XML trees are equal. + Parameters + ---------- + xmltree1 : ~`xml.etree.ElementTree.Element` + The first XML tree for comparison. + xmltree2 : ~`xml.etree.ElementTree.Element` + The second XML tree for comparison. + """ + xml_str1 = etree.tostring(xmltree1).decode("utf-8") + xml_str2 = etree.tostring(xmltree2).decode("utf-8") + checker = XMLOutputChecker() + assert checker.check_output(xml_str1, xml_str2, 0), f"XML trees differ:\n{xml_str1}\n---\n{xml_str2}" + + @staticmethod + def assertXmltreeEqualsFile(xmltree1, xmltree2_file): + """ + Assert that an XML tree is equal to the content of a file. + Parameters + ---------- + xmltree1 : ~`xml.etree.ElementTree.Element` + The XML tree for comparison. + xmltree2_file : str + The path to the file containing the second XML tree. + """ + xmltree2 = XMLOutputChecker.xmltree_from_file(xmltree2_file).getroot() + xml_str1 = etree.tostring(xmltree1).decode("utf-8") + xml_str2 = etree.tostring(xmltree2).decode("utf-8") + checker = XMLOutputChecker() + assert checker.check_output(xml_str1, xml_str2), f"XML trees differ:\n{xml_str1}\n---\n{xml_str2}" diff --git a/pyvo/mivot/tests/data/reference/annotation_seeker.0.1.xml b/pyvo/mivot/tests/data/reference/annotation_seeker.0.1.xml new file mode 100644 index 000000000..34c4a920a --- /dev/null +++ b/pyvo/mivot/tests/data/reference/annotation_seeker.0.1.xml @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pyvo/mivot/tests/data/reference/annotation_seeker.0.2.xml b/pyvo/mivot/tests/data/reference/annotation_seeker.0.2.xml new file mode 100644 index 000000000..8b60bf293 --- /dev/null +++ b/pyvo/mivot/tests/data/reference/annotation_seeker.0.2.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/annotation_seeker.0.4.xml b/pyvo/mivot/tests/data/reference/annotation_seeker.0.4.xml new file mode 100644 index 000000000..8564ee482 --- /dev/null +++ b/pyvo/mivot/tests/data/reference/annotation_seeker.0.4.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/globals_models.json b/pyvo/mivot/tests/data/reference/globals_models.json new file mode 100644 index 000000000..cfdfbbde3 --- /dev/null +++ b/pyvo/mivot/tests/data/reference/globals_models.json @@ -0,0 +1,11 @@ +{ + "COLLECTION": [ + "coords:TimeSys", + "coords:SpaceSys", + "mango:coordinates.PhotometryCoordSys", + "ds:experiment.ObsDataset" + ], + "INSTANCE": [ + "ds:experiment.Target" + ] +} \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/instance_dmtypes.json b/pyvo/mivot/tests/data/reference/instance_dmtypes.json new file mode 100644 index 000000000..516dc7d9e --- /dev/null +++ b/pyvo/mivot/tests/data/reference/instance_dmtypes.json @@ -0,0 +1,36 @@ +{ + "GLOBALS": [ + "coords:TimeSys", + "coords:TimeFrame", + "coords:StdRefLocation", + "coords:SpaceSys", + "coords:SpaceFrame", + "mango:coordinates.PhotometryCoordSys", + "mango:coordinates.PhotFilter", + "mango:coordinates.PhotometryCoordSys", + "mango:coordinates.PhotFilter", + "mango:coordinates.PhotometryCoordSys", + "mango:coordinates.PhotFilter", + "ds:experiment.ObsDataset", + "ds:experiment.Target" + ], + "TEMPLATES": { + "_PKTable": [ + "cube:SparseCube" + ], + "Results": [ + "cube:NDPoint", + "cube:Observable", + "meas:Time", + "coords:MJD", + "cube:Observable", + "meas:GenericMeasure", + "coords:PhysicalCoordinate", + "cube:Observable", + "meas:GenericMeasure", + "coords:PhysicalCoordinate", + "meas:Error", + "meas:Symmetrical" + ] + } +} \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/multiple_templates.xml b/pyvo/mivot/tests/data/reference/multiple_templates.xml new file mode 100644 index 000000000..677103f95 --- /dev/null +++ b/pyvo/mivot/tests/data/reference/multiple_templates.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/static_reference_resolved.xml b/pyvo/mivot/tests/data/reference/static_reference_resolved.xml new file mode 100644 index 000000000..ead465556 --- /dev/null +++ b/pyvo/mivot/tests/data/reference/static_reference_resolved.xml @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/templates_models.json b/pyvo/mivot/tests/data/reference/templates_models.json new file mode 100644 index 000000000..2eb7063ee --- /dev/null +++ b/pyvo/mivot/tests/data/reference/templates_models.json @@ -0,0 +1,25 @@ +{ + "_PKTable": { + "COLLECTION": [], + "INSTANCE": [ + "cube:SparseCube" + ] + }, + "Results": { + "COLLECTION": [], + "INSTANCE": [ + "cube:NDPoint", + "cube:Observable", + "meas:Time", + "coords:MJD", + "cube:Observable", + "meas:GenericMeasure", + "coords:PhysicalCoordinate", + "cube:Observable", + "meas:GenericMeasure", + "coords:PhysicalCoordinate", + "meas:Error", + "meas:Symmetrical" + ] + } +} \ No newline at end of file diff --git a/pyvo/mivot/tests/data/static_reference.xml b/pyvo/mivot/tests/data/static_reference.xml new file mode 100644 index 000000000..776b7c0c6 --- /dev/null +++ b/pyvo/mivot/tests/data/static_reference.xml @@ -0,0 +1,8 @@ + + + + + + + \ No newline at end of file diff --git a/pyvo/mivot/tests/data/test.mivot_viewer.first_instance.xml b/pyvo/mivot/tests/data/test.mivot_viewer.first_instance.xml new file mode 100644 index 000000000..2e90f7f4d --- /dev/null +++ b/pyvo/mivot/tests/data/test.mivot_viewer.first_instance.xml @@ -0,0 +1,45 @@ + + + + Prototype for covariance errors + + + hand-made mapping + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SourceName
OtherSourceName
+ + \ No newline at end of file diff --git a/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml b/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml new file mode 100644 index 000000000..54c5b32f5 --- /dev/null +++ b/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml @@ -0,0 +1,222 @@ + + + + + URAT1 Catalog (Zacharias+ 2015) IVOID of underlying data collection + + + + + URAT1 catalog + + + + Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of catalog (Epoch) + + + + URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] + + + + Right ascension on ICRS, at "Epoch" (1) + + + + Declination on ICRS, at "Epoch" (1) + + + + Position error per coordinate, from scatter (2) + + + + Position error per coordinate, from model (2) + + + + (nst) Total number of sets the star is in (3) + + + + (nsu) Number of sets used for mean position (3) + + + + (epoc) Mean URAT observation epoch (1) + + + + ?(mmag) mean URAT model fit magnitude (4) + + + + + ?(sigp) URAT photometry error (5) + + + + + (nsm) Number of sets used for URAT magnitude (3) + + + + (ref) largest reference star flag (6) + + + + (nit) Total number of images (observations) + + + + (niu) Number of images used for mean position + + + + (ngt) Total number of 1st order grating observations + + + + (ngu) Number of 1st order grating positions used + + + + ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) + + + + + ?(pmd) Proper motion in Declination (7) + + + + + ?(pme) Proper motion error per coordinate (8) + + + + + [1/11] Match flag URAT with 2MASS (9) + + + + [1/11] Match flag URAT with APASS (9) + + + + [-] "-" if there is no match with GSC2.4 (14) + + + + ?(id2) unique 2MASS star identification number + + + + + ?(jmag) 2MASS J-band magnitude + + + + + ?(ejmag) Error on Jmag + + + + + [0,58]? J-band quality-confusion flag (10) + + + + ?(hmag) 2MASS H-band magnitude + + + + + ?(ehmag) Error on H-band magnitude (10) + + + + + [0,58]? H-band quality-confusion flag (10) + + + + ?(kmag) 2MASS Ks-band magnitude + + + + + ?(ekmag) Error on Ks-band magnitude (10) + + + + + [0,58]? Ks-band quality-confusion flag (10) + + + + (ann) Number of APASS observation nights (12) + + + + (ano) Number of APASS observations (12) + + + + ?(abm) APASS B-band magnitude (11) + + + + + ?(ebm) Error on Bmag + + + + + ?(avm) APASS V-band magnitude + + + + + ?(evm) Error on Vmag + + + + + ?(agm) APASS g-band magnitude + + + + + ?(egm) Error on gmag + + + + + ?(arm) APASS r-band magnitude + + + + + ?(erm) Error on rmag + + + + + ?(aim) APASS i-band magnitude + + + + + ?(eim) Error on imag + + + + + + +
0.049402750-146023052.2340018+59.89373339613132013.41815.3400.0131397474001.5-12.35.91575880868113.7130.028513.3400.034513.1010.03451417.6320.20416.1640.00116.6900.00115.7500.001
+ + diff --git a/pyvo/mivot/tests/data/test.mivot_viewer.xml b/pyvo/mivot/tests/data/test.mivot_viewer.xml new file mode 100644 index 000000000..ef5238c6e --- /dev/null +++ b/pyvo/mivot/tests/data/test.mivot_viewer.xml @@ -0,0 +1,303 @@ + + + Epoch photometry. This table contains light curve data points. Each entry is a photometric flux-time pair for a given object, band and time. It follows the (evolving) VO convention for time series as much as possible. At the time of writing, a VO recommendation has not yet been released. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Source ID + + + Photometric Band + + + + + + + + + + + + + + + + + +
5813181197970338560G
5813181197970338560BP
5813181197970338560RP
+ + +Source Id. A unique single numerical identifier of the source obtained from GaiaSource. + + +Transit unique identifier. For a given object, a transit comprises the different Gaia observations (SM, AF, BP, RP and RVS) obtained for each focal plane crossing. + + +Photometric band. Values: G (per-transit combined SM-AF flux), BP (blue photometer integrated flux) and RP (red photometer integrated flux). + + +Different times are defined for each band. For G, it is the field-of-view transit averaged observation time. For BP and RP, it is the observation time of the BP CCD transit. The units are Barycentric JD (in TCB) in days -2,455,197.5, computed as follows. First the observation time is converted from On-board Mission Time (OBMT) into Julian date in TCB (Temps Coordonnee Barycentrique). Next a correction is applied for the light-travel time to the Solar system barycentre, resulting in Barycentric Julian Date (BJD). Finally, an offset of 2,455,197.5 days is applied (corresponding to a reference time $T_0$ at 2010-01-01T00:00:00) to have a conveniently small numerical value. Although the centroiding time accuracy of the individual CCD observations is (much) below 1~ms (e.g. in BP and RP), the G band observation time is averaged over typically 9 CCD observations taken in a time range of about 44sec. + + +Vega magnitude, computed from the flux applying the zero-point defined in ExtPhotZeroPoint. + + +Band flux value for the transit. For G band, it is a combination of individual SM-AF CCD fluxes. For BP and RP bands, it is an integrated CCD flux. + + +Flux error. If the flux has been rejected or is unavailable, this error will be set to null. + + +Band flux divided by its error. If the flux has been rejected or is unavailable, this field will be set to null. + + +Rejected by DPAC photometry processing. + + +Rejected by DPAC variability processing (or variability analysis). + + + + +All Gaia data processed by the Data Processing and Analysis Consortium comes tagged with a solution identifier. This is a numeric field attached to each table row that can be used to unequivocally identify the version of all the subsystems that where used in the generation of the data as well as the input data used. It is mainly for internal DPAC use but is included in the published data releases to enable end users to examine the provenance of processed data products. To decode a given solution ID visit https://gaia.esac.esa.int/decoder/solnDecoder.jsp + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
581318119797033856017091923342681275G1705.943736020098415.21657477445216415442.45627327361644.151258712309364349.76254FF4097369295551293819386
581318119797033856017096015648964756G1706.017710021738614.76733669360487723356.7069931982333.53035403015752696.584FF4194817369295551293819386
581318119797033856019103616164443503G1742.321576336688615.27834299913750214588.44795624094115.054069973748831969.07FF1369295551293819386
+ + diff --git a/pyvo/mivot/tests/test_annotation_seeker.py b/pyvo/mivot/tests/test_annotation_seeker.py new file mode 100644 index 000000000..4b3be15a1 --- /dev/null +++ b/pyvo/mivot/tests/test_annotation_seeker.py @@ -0,0 +1,92 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Test for mivot.seekers.annotation_seeker.py +""" +import os +import pytest +try: + from defusedxml import ElementTree as etree +except ImportError: + from xml.etree import ElementTree as etree +from pyvo.mivot.seekers.annotation_seeker import AnnotationSeeker +from pyvo.mivot.utils.dict_utils import DictUtils +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot import MivotViewer +from . import XMLOutputChecker + + +@pytest.fixture +def a_seeker(data_path,): + m_viewer = MivotViewer(os.path.join(data_path, "data", "test.mivot_viewer.xml"), + tableref="Results") + return AnnotationSeeker(m_viewer._mapping_block) + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_multiple_templates(data_path): + """ + Try to create an AnnotationSeeker with a mapping_block containing multiple TEMPLATES. + """ + mapping_block = XMLOutputChecker.xmltree_from_file( + os.path.join(data_path, "data/reference/multiple_templates.xml")) + with pytest.raises(Exception, match="TEMPLATES without tableref must be unique"): + AnnotationSeeker(mapping_block.getroot()) + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_all_reverts(a_seeker, data_path): + # Checks the GLOBALS block given by the AnnotationSeeker + # by comparing it to the content of the file test.0.1.xml + XMLOutputChecker.assertXmltreeEqualsFile(a_seeker.globals_block, + os.path.join(data_path, "data/reference/annotation_seeker.0.1.xml")) + # Checks the TEMPLATES block given by the AnnotationSeeker + # by comparing it to the content of the file test.0.2.xml + XMLOutputChecker.assertXmltreeEqualsFile(a_seeker.get_templates_block("Results"), + os.path.join(data_path, "data/reference/annotation_seeker.0.2.xml")) + # Checks the list of all the tableref found by the AnnotationSeeker + assert list(a_seeker.get_templates_tableref()) == ['_PKTable', 'Results'] + # a_seeker should have only 2 COLLECTIONS in GLOBALS: _CoordinateSystems and _Datasets + assert len(a_seeker.get_globals_collections()) == 2 + # a_seeker should have only 1 INSTANCES in GLOBALS: _tg1 + assert len(a_seeker.get_globals_instances()) == 1 + assert a_seeker.get_globals_instance_dmtypes() == ['ds:experiment.Target'] + assert (a_seeker.get_globals_instance_dmids() + == ['_timesys', '_spacesys1', '_photsys_G', '_photsys_RP', + '_photsys_BP', '_ds1', '_tg1']) + assert a_seeker.get_globals_collection_dmids() == ['_CoordinateSystems', '_Datasets'] + selection = a_seeker.get_instance_by_dmtype("coords") + assert len(selection["GLOBALS"]) == 5 + for ele in selection["GLOBALS"]: + assert ele.get("dmtype").startswith("coords") + assert len(selection["TEMPLATES"]["_PKTable"]) == 0 + assert len(selection["TEMPLATES"]["Results"]) == 3 + for _, table_sel in selection["TEMPLATES"].items(): + for ele in table_sel: + assert ele.get("dmtype").startswith("coords") + with pytest.raises(Exception, match="INSTANCE with PRIMARY_KEY = wrong_key_value " + "in COLLECTION dmid wrong_key_value not found"): + a_seeker.get_collection_item_by_primarykey("_Datasets", "wrong_key_value") + pksel = a_seeker.get_collection_item_by_primarykey("_Datasets", "5813181197970338560") + XMLOutputChecker.assertXmltreeEqualsFile(pksel, + os.path.join(data_path, "data/reference/annotation_seeker.0.4.xml")) + with (pytest.raises(Exception, match="More than one INSTANCE with " + "PRIMARY_KEY = G found in COLLECTION dmid G")): + double_key = etree.fromstring("""""") + a_seeker.get_collection_item_by_primarykey("_CoordinateSystems", "G" + ).append(double_key) + a_seeker.get_collection_item_by_primarykey("_CoordinateSystems", "G") + with pytest.raises(Exception, match="INSTANCE with PRIMARY_KEY = wrong_key " + "in COLLECTION dmid wrong_key not found"): + a_seeker.get_collection_item_by_primarykey("_CoordinateSystems", "wrong_key") + assert a_seeker.get_instance_dmtypes() == DictUtils.read_dict_from_file( + os.path.join(data_path, "data/reference/instance_dmtypes.json")) + assert a_seeker.get_templates_instance_by_dmid("Results", "wrong_dmid") is None + assert a_seeker.get_templates_instance_by_dmid("Results", "_ts_data").get("dmtype") == "cube:NDPoint" + assert a_seeker.get_globals_instance_from_collection( + "_CoordinateSystems", "ICRS").get("dmtype") == "coords:SpaceSys" + assert a_seeker.get_globals_instance_from_collection("wrong_dmid", "ICRS") is None diff --git a/pyvo/mivot/tests/test_mivot_instance.py b/pyvo/mivot/tests/test_mivot_instance.py new file mode 100644 index 000000000..8feedf2c8 --- /dev/null +++ b/pyvo/mivot/tests/test_mivot_instance.py @@ -0,0 +1,81 @@ +''' +Test the class generation from a dict.x +Created on 19 févr. 2024 + +@author: michel +''' +import pytest +from astropy.table import Table +from pyvo.mivot.viewer.mivot_instance import MivotInstance + + +fake_hk_dict = { + "dmtype": "EpochPosition", + "longitude": { + "dmtype": "RealQuantity", + "value": 52.2340018, + "unit": "deg", + "astropy_unit": {}, + "ref": "RAICRS" + }, + "latitude": { + "dmtype": "RealQuantity", + "value": 59.8937333, + "unit": "deg", + "astropy_unit": {}, + "ref": "DEICRS" + } +} + +fake_dict = { + "dmtype": "EpochPosition", + "longitude": { + "value": 52.2340018, + "unit": "deg", + }, + "latitude": { + "value": 59.8937333, + "unit": "deg", + } +} + + +def test_mivot_instance_constructor(): + """Test the class generation from a dict.""" + mivot_object = MivotInstance(**fake_hk_dict) + assert mivot_object.longitude.value == 52.2340018 + assert mivot_object.longitude.unit == "deg" + assert mivot_object.latitude.value == 59.8937333 + assert mivot_object.latitude.unit == "deg" + assert mivot_object.longitude.dmtype == "RealQuantity" + assert mivot_object.dmtype == "EpochPosition" + + +def test_mivot_instance_update(): + """Test the class generation from a dict followed by an update""" + mivot_object = MivotInstance(**fake_hk_dict) + + t = Table() + t["RAICRS"] = [67.87] + t["DEICRS"] = [-89.87] + mivot_object.update(t[0]) + assert mivot_object.longitude.value == 67.87 + assert mivot_object.latitude.value == -89.87 + + +def test_mivot_instance_update_wrong_columns(): + """Test the class generation from a dict followed by an update with wrong columns.""" + mivot_object = MivotInstance(**fake_hk_dict) + + t = Table() + t["RAICRSXX"] = [67.87] + t["DEICRS"] = [-89.87] + with pytest.raises(KeyError, match="RAICRS"): + mivot_object.update(t[0]) + + +def test_mivot_instance_display_dict(): + """Test the class generation from a dict and rebuild the dict from the instance.""" + mivot_object = MivotInstance(**fake_hk_dict) + assert mivot_object.hk_dict == fake_hk_dict + assert mivot_object.dict == fake_dict diff --git a/pyvo/mivot/tests/test_mivot_instance_generation.py b/pyvo/mivot/tests/test_mivot_instance_generation.py new file mode 100644 index 000000000..026d45762 --- /dev/null +++ b/pyvo/mivot/tests/test_mivot_instance_generation.py @@ -0,0 +1,179 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Test for mivot.viewer.model_viewer_level3.py and mivot.viewer.mivot_time.py +""" +import os +import pytest +from urllib.request import urlretrieve +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot import MivotViewer +from pyvo.mivot.utils.mivot_utils import MivotUtils + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_model_viewer3(votable_test, simple_votable): + """ + Recursively compare an XML element with an element of MIVOT + class with the function recursive_xml_check. + This test run on 2 votables : votable_test and simple_votable. + """ + m_viewer_simple_votable = MivotViewer(votable_path=simple_votable) + MivotInstance = m_viewer_simple_votable.dm_instance + xml_simple_votable = m_viewer_simple_votable.xml_view + assert xml_simple_votable.tag == 'TEMPLATES' + recusive_xml_check(xml_simple_votable, MivotInstance) + m_viewer_votable_test = MivotViewer(votable_path=votable_test) + m_viewer_votable_test.next() + mivot_instance = m_viewer_votable_test.dm_instance + xml_votable_test = m_viewer_votable_test.xml_view + assert xml_simple_votable.tag == 'TEMPLATES' + recusive_xml_check(xml_votable_test, mivot_instance) + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def recusive_xml_check(xml_simple_votable, MivotInstance): + if xml_simple_votable.tag == 'TEMPLATES': + recusive_xml_check(xml_simple_votable[0], MivotInstance) + else: + for child in xml_simple_votable: + if child.tag == 'INSTANCE': + for key, value in child.attrib.items(): + if key == 'dmrole': + if value == '': + if child.tag == 'ATTRIBUTE': + recusive_xml_check(child, + getattr(MivotInstance, + MivotInstance._remove_model_name( + child.get('dmrole')))) + elif child.tag == 'INSTANCE': + recusive_xml_check(child, getattr(MivotInstance, + MivotInstance._remove_model_name + (child.get('dmrole'), True))) + else: + if child.tag == 'ATTRIBUTE': + recusive_xml_check(child, getattr(MivotInstance, + MivotInstance._remove_model_name( + child.get('dmrole')))) + elif child.tag == 'INSTANCE': + recusive_xml_check(child, getattr(MivotInstance, + MivotInstance._remove_model_name( + child.get('dmrole'), True))) + elif child.tag == 'COLLECTION': + recusive_xml_check(child, getattr(MivotInstance, + MivotInstance._remove_model_name( + child.get('dmrole')))) + elif child.tag == 'COLLECTION': + for key, value in child.attrib.items(): + assert len(getattr(MivotInstance, + MivotInstance._remove_model_name(child.get('dmrole')))) == len(child) + i = 0 + for child2 in child: + recusive_xml_check(child2, getattr(MivotInstance, MivotInstance._remove_model_name + (child.get('dmrole')))[i]) + i += 1 + elif child.tag == 'ATTRIBUTE': + MivotInstance_attribute = getattr(MivotInstance, + MivotInstance._remove_model_name(child.get('dmrole'))) + for key, value in child.attrib.items(): + if key == 'dmtype': + assert MivotInstance_attribute.dmtype in value + elif key == 'value': + if (MivotInstance_attribute.value is not None + and not isinstance(MivotInstance_attribute.value, bool)): + if isinstance(MivotInstance_attribute.value, float): + pytest.approx(float(value), MivotInstance_attribute.value, 0.0001) + else: + assert value == MivotInstance_attribute.value + else: + assert False + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_dict_model_viewer3(votable_test, simple_votable): + """ + To test the generation of the MIVOT class, the function builds a ModelViewerLevel3 + with his MIVOT class and his previous dictionary from XML. + Then, it calls the function recursive_check which recursively compares an element of MIVOT class + with the dictionary on which it was built. + MIVOT class is itself a dictionary with only essential information of the ModelViewerLevel3._dict. + This test run on 2 votables : votable_test and simple_votable. + """ + m_viewer_votable_test = MivotViewer(votable_path=votable_test) + m_viewer_votable_test.next() + mivot_instance = m_viewer_votable_test.dm_instance + _dict = MivotUtils.xml_to_dict(m_viewer_votable_test.xml_viewer.view) + recursive_check(mivot_instance, **_dict) + mivot_instance = m_viewer_votable_test.dm_instance + _dict = MivotUtils.xml_to_dict(m_viewer_votable_test.xml_view) + recursive_check(mivot_instance, **_dict) + + +def recursive_check(MivotInstance, **kwargs): + for key, value in kwargs.items(): + # the root instance ha no role: this makes an empty value in the unpacked dict + if key == '': + continue + if isinstance(value, list): + nbr_item = 0 + for item in value: + if isinstance(item, dict): + assert 'dmtype' in item.keys() + recursive_check(getattr(MivotInstance, + MivotInstance._remove_model_name(key))[nbr_item], + **item + ) + nbr_item += 1 + elif isinstance(value, dict) and 'value' not in value: + # for INSTANCE of INSTANCEs dmrole needs model_name + assert MivotInstance._remove_model_name(key, True) in vars(MivotInstance).keys() + recursive_check(getattr(MivotInstance, MivotInstance._remove_model_name(key, True)), **value) + else: + if isinstance(value, dict) and MivotInstance._is_leaf(**value): + assert value.keys().__contains__('dmtype' and 'value' and 'unit' and 'ref') + lower_dmtype = value['dmtype'].lower() + if "real" in lower_dmtype or "double" in lower_dmtype or "float" in lower_dmtype: + assert isinstance(value['value'], float) + elif "bool" in lower_dmtype: + assert isinstance(value['value'], bool) + elif value['dmtype'] is None: + assert (value['value'] in + ('notset', 'noset', 'null', 'none', 'NotSet', 'NoSet', 'Null', 'None')) + else: + if value['value'] is not None: + assert isinstance(value['value'], str) + recursive_check(getattr(MivotInstance, MivotInstance._remove_model_name(key)), **value) + else: + assert key == 'dmtype' or 'value' + + +@pytest.fixture +def votable_test(data_path, data_sample_url): + votable_name = "vizier_csc2_gal.annot.xml" + votable_path = os.path.join(data_path, "data", votable_name) + urlretrieve(data_sample_url + votable_name, + votable_path) + yield votable_path + os.remove(votable_path) + + +@pytest.fixture +def simple_votable(data_path, data_sample_url): + votable_name = "simple-annotation-votable.xml" + votable_path = os.path.join(data_path, "data", votable_name) + urlretrieve(data_sample_url + votable_name, + votable_path) + yield votable_path + os.remove(votable_path) + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) + + +@pytest.fixture +def data_sample_url(): + return "https://raw.githubusercontent.com/ivoa/dm-usecases/main/pyvo-ci-sample/" diff --git a/pyvo/mivot/tests/test_mivot_viewer.py b/pyvo/mivot/tests/test_mivot_viewer.py new file mode 100644 index 000000000..f14d280f0 --- /dev/null +++ b/pyvo/mivot/tests/test_mivot_viewer.py @@ -0,0 +1,142 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Test for mivot.viewer.mivot_viewer.py +""" +import os +import pytest +import re +from pyvo.mivot.utils.vocabulary import Constant +from pyvo.mivot.utils.dict_utils import DictUtils +from pyvo.mivot.utils.exceptions import MappingException +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot import MivotViewer +from astropy import version as astropy_version + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_get_first_instance_dmtype(path_to_first_instance): + """ + Test the function get_first_instance_dmtype() which is + used to find the first INSTANCE/COLLECTION in TEMPLATES. + """ + m_viewer = MivotViewer(votable_path=path_to_first_instance) + assert m_viewer.get_first_instance_dmtype("one_instance") == "one_instance" + assert m_viewer.get_first_instance_dmtype("coll_and_instances") == "first" + assert m_viewer.get_first_instance_dmtype("one_collection") == Constant.ROOT_COLLECTION + assert m_viewer.get_first_instance_dmtype("only_collection") == Constant.ROOT_COLLECTION + with pytest.raises(Exception, match="Can't find the first INSTANCE/COLLECTION in TEMPLATES"): + m_viewer.get_first_instance_dmtype("empty") + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_table_ref(m_viewer): + """ + Test if the mivot_viewer can find each table_ref and connect to the right table_ref. + Test if the mivot_viewer can find each models. + """ + assert m_viewer._mapped_tables == ['_PKTable', 'Results'] + with pytest.raises(Exception, + match=re.escape(r"The table first_table doesn't match with any mapped_table " + r"(['_PKTable', 'Results']) encountered in TEMPLATES")): + m_viewer._connect_table("wrong_tableref") + assert m_viewer.connected_table_ref == Constant.FIRST_TABLE + assert (m_viewer.get_models() + == {'mango': 'file:/Users/sao/Documents/IVOA/GitHub/ivoa-dm-examples/tmp/Mango-v1.0.vo-dml.xml', + 'cube': 'https://volute.g-vo.org/svn/trunk/projects/dm/Cube/vo-dml/Cube-1.0.vo-dml.xml', + 'ds': 'https://volute.g-vo.org/svn/trunk/projects/dm/' + 'DatasetMetadata/vo-dml/DatasetMetadata-1.0.vo-dml.xml', + 'meas': 'https://www.ivoa.net/xml/Meas/20200908/Meas-v1.0.vo-dml.xml', + 'coords': 'https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml', + 'ivoa': 'https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml'}) + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_global_getters(m_viewer, data_path): + """ + Test each getter of the model_viewer_level1 specific for the GLOBALS. + """ + assert m_viewer.get_table_ids() == ['_PKTable', 'Results'] + assert m_viewer.get_globals_models() == DictUtils.read_dict_from_file( + os.path.join(data_path, "data/reference/globals_models.json")) + assert m_viewer.get_templates_models() == DictUtils.read_dict_from_file( + os.path.join(data_path, "data/reference/templates_models.json")) + m_viewer._connect_table('_PKTable') + row = m_viewer.next_table_row() + assert row[0] == '5813181197970338560' + assert row[1] == 'G' + row = m_viewer.next_table_row() + assert row[0] == '5813181197970338560' + assert row[1] == 'BP' + m_viewer.rewind() + row = m_viewer.next_table_row() + assert row[0] == '5813181197970338560' + assert row[1] == 'G' + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_no_mivot(path_no_mivot): + """ + Test each getter of the model_viewer_level1 specific for the GLOBALS. + """ + m_viewer = MivotViewer(path_no_mivot) + assert m_viewer.get_table_ids() is None + assert m_viewer.get_globals_models() is None + + assert m_viewer.get_templates_models() is None + with pytest.raises(MappingException): + m_viewer._connect_table('_PKTable') + with pytest.raises(MappingException): + m_viewer._connect_table() + + assert m_viewer.next_table_row() is None + + +def test_check_version(path_to_viewer): + if not check_astropy_version(): + with pytest.raises(Exception, + match=f"Astropy version {astropy_version.version} " + f"is below the required version 6.0 for the use of MIVOT."): + MivotViewer(votable_path=path_to_viewer) + if astropy_version.version is None: + assert not check_astropy_version() + elif astropy_version.version < '6.0': + assert not check_astropy_version() + else: + assert check_astropy_version() is True + + +@pytest.fixture +def m_viewer(data_path): + if not check_astropy_version(): + pytest.skip("MIVOT test skipped because of the astropy version.") + + votable_name = "test.mivot_viewer.xml" + votable_path = os.path.join(data_path, "data", votable_name) + return MivotViewer(votable_path=votable_path) + + +@pytest.fixture +def path_to_viewer(data_path): + if not check_astropy_version(): + pytest.skip("MIVOT test skipped because of the astropy version.") + + votable_name = "test.mivot_viewer.xml" + return os.path.join(data_path, "data", votable_name) + + +@pytest.fixture +def path_to_first_instance(data_path): + + votable_name = "test.mivot_viewer.first_instance.xml" + return os.path.join(data_path, "data", votable_name) + + +@pytest.fixture +def path_no_mivot(data_path): + votable_name = "test.mivot_viewer.no_mivot.xml" + return os.path.join(data_path, "data", votable_name) + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) diff --git a/pyvo/mivot/tests/test_resource_seeker.py b/pyvo/mivot/tests/test_resource_seeker.py new file mode 100644 index 000000000..9647aab44 --- /dev/null +++ b/pyvo/mivot/tests/test_resource_seeker.py @@ -0,0 +1,73 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Test for mivot.seekers.resource_seeker.py +""" +import os +import pytest +from astropy.io.votable import parse +from pyvo.mivot.seekers.resource_seeker import ResourceSeeker +from pyvo.mivot.version_checker import check_astropy_version + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_id_table(rseeker): + """ + Checks the IDs of tables found by the RessourceSeeker, + checks the IDs of the field of the table concerned. + """ + assert rseeker.get_table_ids() == ['_PKTable', 'Results'] + assert (rseeker.get_id_index_mapping('_PKTable') + == {'pksrcid': {'ID': '_pksrcid', 'indx': 0}, + 'pkband': {'ID': '_pkband', 'indx': 1} + } + ) + assert (rseeker.get_id_index_mapping('Results') + == {'source_id': {'ID': '_srcid', 'indx': 0}, + 'transit_id': {'ID': 'transit_id', 'indx': 1}, + 'band': {'ID': '_band', 'indx': 2}, + 'time': {'ID': '_obstime', 'indx': 3}, + 'mag': {'ID': '_mag', 'indx': 4}, + 'flux': {'ID': '_flux', 'indx': 5}, + 'flux_error': {'ID': '_fluxerr', 'indx': 6}, + 'flux_over_error': {'ID': 'flux_over_error', 'indx': 7}, + 'rejected_by_photometry': {'ID': 'rejected_by_photometry', 'indx': 8}, + 'rejected_by_variability': {'ID': 'rejected_by_variability', 'indx': 9}, + 'other_flags': {'ID': 'other_flags', 'indx': 10}, + 'solution_id': {'ID': 'solution_id', 'indx': 11} + } + ) + table = rseeker.get_table('_PKTable') + for field in table.fields: + field.ID = None + assert (rseeker.get_id_index_mapping('_PKTable') + == {'pksrcid': {'indx': 0, 'ID': 'pksrcid'}, + 'pkband': {'indx': 1, 'ID': 'pkband'} + } + ) + for table in rseeker._resource.tables: + table.ID = None + assert rseeker.get_table_ids() == ['AnonymousTable', 'AnonymousTable'] + + for table in rseeker._resource.tables: + table.name = "any_name" + + assert rseeker.get_table_ids() == ['any_name', 'any_name'] + assert (rseeker.get_id_index_mapping('any_name') + == {'pksrcid': {'indx': 0, 'ID': 'pksrcid'}, + 'pkband': {'indx': 1, 'ID': 'pkband'}} + ) + + +@pytest.fixture +def rseeker(data_path): + + votable_path = os.path.join(data_path, "data", "test.mivot_viewer.xml") + + votable = parse(votable_path) + for resource in votable.resources: + return ResourceSeeker(resource) + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) diff --git a/pyvo/mivot/tests/test_static_reference.py b/pyvo/mivot/tests/test_static_reference.py new file mode 100644 index 000000000..058cdd44a --- /dev/null +++ b/pyvo/mivot/tests/test_static_reference.py @@ -0,0 +1,38 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Test for mivot.features.static_reference_resolver.py +""" +import os +import pytest +from pyvo.mivot.seekers.annotation_seeker import AnnotationSeeker +from pyvo.mivot.features.static_reference_resolver import StaticReferenceResolver +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot import MivotViewer +from . import XMLOutputChecker + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_static_reference_resolve(a_seeker, instance, data_path): + StaticReferenceResolver.resolve(a_seeker, None, instance) + XMLOutputChecker.assertXmltreeEqualsFile(instance.getroot(), + os.path.join(data_path, + "data/reference/static_reference_resolved.xml")) + + +@pytest.fixture +def instance(data_path): + return XMLOutputChecker.xmltree_from_file(os.path.join( + data_path, + "data/static_reference.xml")) + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) + + +@pytest.fixture +def a_seeker(data_path): + m_viewer = MivotViewer(os.path.join(data_path, "data", "test.mivot_viewer.xml"), + tableref="Results") + return AnnotationSeeker(m_viewer._mapping_block) diff --git a/pyvo/mivot/tests/test_user_api.py b/pyvo/mivot/tests/test_user_api.py new file mode 100644 index 000000000..f6091e9f4 --- /dev/null +++ b/pyvo/mivot/tests/test_user_api.py @@ -0,0 +1,382 @@ +""" +Check the API exposed to the user works properly +This unit test can also be used as a code example +Created on 26 Feb. 2024 +@author: michel +""" + +import os +import pytest +from urllib.request import urlretrieve +import astropy.units as u +from astropy.coordinates import SkyCoord +from pyvo.dal.scs import SCSService +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot import MivotViewer +from astropy.io.votable import parse +from pyvo.mivot.utils.dict_utils import DictUtils + +ref_ra = [ + 0.04827189, + 0.16283175, + 0.29222255, + 0.42674592, + 359.5190115, + 359.94372764, +] +ref_dec = [ + -0.36042119, + 0.22293899, + -0.07592034, + -0.21749947, + -0.1281483, + -0.28005255, +] +ref_pmdec = [ + -11.67, + -3.09, + -73.28, + -114.08, + -19.05, + -25.43 +] +ref_pmra = [ + 61.75, + 39.02, + 54.94, + 20.73, + -45.19, + -5.14 +] + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) + + +@pytest.fixture +def data_sample_url(): + return "https://raw.githubusercontent.com/ivoa/dm-usecases/main/pyvo-ci-sample/" + + +@pytest.fixture +def vizier_url(): + return "https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch/I/239/hip_main" + + +@pytest.fixture +def delt_coo(): + """acceptable delta for coordinate value comparisons""" + return 0.0000001 + + +@pytest.fixture +def path_to_votable(data_path, data_sample_url): + + votable_name = "vizier_for_user_api.xml" + votable_path = os.path.join(data_path, "data", votable_name) + urlretrieve(data_sample_url + votable_name, votable_path) + + yield votable_path + os.remove(votable_path) + + +@pytest.fixture +def path_to_full_mapped_votable(data_path, data_sample_url): + + votable_name = "gaia_epoch_propagation_full.xml" + votable_path = os.path.join(data_path, "data", votable_name) + urlretrieve(data_sample_url + votable_name, votable_path) + + yield votable_path + os.remove(votable_path) + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_mivot_viewer_next(path_to_votable): + """ + Check that ten MivotViewer iterating over the data rows provides the expected values + """ + mivot_viewer = MivotViewer(path_to_votable) + mivot_instance = mivot_viewer.dm_instance + assert mivot_instance.dmtype == "EpochPosition" + assert mivot_instance.Coordinate_coordSys.spaceRefFrame.value == "ICRS" + ra = [] + dec = [] + pmra = [] + pmdec = [] + while mivot_viewer.next(): + ra.append(mivot_instance.longitude.value) + dec.append(mivot_instance.latitude.value) + pmra.append(mivot_instance.pmLongitude.value) + pmdec.append(mivot_instance.pmLatitude.value) + assert ra == ref_ra + assert dec == ref_dec + assert pmra == ref_pmra + assert pmdec == ref_pmdec + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_mivot_tablerow_next(path_to_votable): + """ + Check that the MIVOT interpreter can be applied to a classical table readout. + - The MivotViewer is initialized on the first data row (behind the stage) + to be able to build a MivoInstance that will provide a model view on the data + - The data table is read in a classical way + - The MivotInstance is updated with data row providing so a model view on it + """ + votable = parse(path_to_votable) + table = votable.resources[0].tables[0] + mivot_viewer = MivotViewer(votable) + + mivot_instance = mivot_viewer.dm_instance + assert mivot_instance.dmtype == "EpochPosition" + assert mivot_instance.Coordinate_coordSys.spaceRefFrame.value == "ICRS" + ra = [] + dec = [] + pmra = [] + pmdec = [] + for rec in table.array: + mivot_instance.update(rec) + ra.append(mivot_instance.longitude.value) + dec.append(mivot_instance.latitude.value) + pmra.append(mivot_instance.pmLongitude.value) + pmdec.append(mivot_instance.pmLatitude.value) + assert ra == ref_ra + assert dec == ref_dec + assert pmra == ref_pmra + assert pmdec == ref_pmdec + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_external_iterator(path_to_votable, delt_coo): + """Checks that the values returned by MIVOT are + the same as those read by Astropy + - The MivotViewer is initialized on the first data row (behind the stage) + to be able to build a MivoInstance that will provide a model view on the data + - The data table is read in a classical way + - The MivotInstance is updated with data row providing so a model view on it + - The attribute values are then checked against the table data + """ + # parse the VOTable outside of the viewer + votable = parse(path_to_votable) + table = votable.resources[0].tables[0] + # init the viewer + mivot_viewer = MivotViewer(votable) + mivot_instance = mivot_viewer.dm_instance + for rec in table.array: + mivot_instance.update(rec) + assert rec["RAICRS"] == mivot_instance.longitude.value + assert rec["DEICRS"] == mivot_instance.latitude.value + assert rec["pmRA"] == mivot_instance.pmLongitude.value + assert rec["pmDE"] == mivot_instance.pmLatitude.value + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_with_withstatement(path_to_votable): + """check that the values read by a MivotViewer used with a with statement + are those expected + """ + + read_ra = [] + read_dec = [] + read_pmra = [] + read_pmdec = [] + with MivotViewer(path_to_votable) as mivot_viewer: + mivot_object = mivot_viewer.dm_instance + while mivot_viewer.next(): + read_ra.append(mivot_object.longitude.value) + read_dec.append(mivot_object.latitude.value) + read_pmra.append(mivot_object.pmLongitude.value) + read_pmdec.append(mivot_object.pmLatitude.value) + assert read_ra == ref_ra + assert read_dec == ref_dec + assert read_pmra == ref_pmra + assert read_pmdec == ref_pmdec + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_with_dict(path_to_votable): + """check that the MIVOT object user dictionary match the read data""" + + with MivotViewer(path_to_votable) as mivot_viewer: + mivot_object = mivot_viewer.dm_instance + # let's focus on the last data row + while mivot_viewer.next(): + pass + + # check the slim (user friendly) dictionary + assert mivot_object.dict == { + "dmtype": "EpochPosition", + "longitude": {"value": 359.94372764, "unit": "deg"}, + "latitude": {"value": -0.28005255, "unit": "deg"}, + "pmLongitude": {"value": -5.14, "unit": "mas/yr"}, + "pmLatitude": {"value": -25.43, "unit": "mas/yr"}, + "epoch": {"value": 1991.25, "unit": "year"}, + "Coordinate_coordSys": { + "dmtype": "SpaceSys", + "dmid": "SpaceFrame_ICRS", + "dmrole": "coordSys", + "spaceRefFrame": {"value": "ICRS"}, + }, + } + # check the whole dictionary + assert mivot_object.hk_dict == { + "dmtype": "EpochPosition", + "longitude": {"dmtype": "RealQuantity", "value": 359.94372764, + "unit": "deg", "astropy_unit": {}, "ref": "RAICRS"}, + "latitude": {"dmtype": "RealQuantity", "value": -0.28005255, + "unit": "deg", "astropy_unit": {}, "ref": "DEICRS"}, + "pmLongitude": {"value": -5.14, "unit": "mas/yr", "dmtype": "RealQuantity", + "ref": "pmRA", "astropy_unit": {}}, + "pmLatitude": {"value": -25.43, "unit": "mas/yr", "dmtype": "RealQuantity", + "ref": "pmDE", "astropy_unit": {}}, + "epoch": {"dmtype": "RealQuantity", "ref": None, "unit": "year", "value": 1991.25}, + "Coordinate_coordSys": { + "dmtype": "SpaceSys", + "dmid": "SpaceFrame_ICRS", + "dmrole": "coordSys", + "spaceRefFrame": {"dmtype": "SpaceFrame", "ref": None, "unit": None, "value": "ICRS"}, + }, + } + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_with_full_dict(path_to_full_mapped_votable): + """check that the MIVOT object user dictionary match the read data + - The data sample used here maps the whole EpochPosition model except TimeSys + """ + + with MivotViewer(path_to_full_mapped_votable) as mivot_viewer: + mivot_object = mivot_viewer.dm_instance + # let's focus on the second data row + while mivot_viewer.next(): + DictUtils.print_pretty_json(mivot_object.dict) + # check the slim (user friendly) dictionary + assert mivot_object.dict == { + "dmtype": "EpochPosition", + "longitude": {"value": 307.79115807079, "unit": "deg"}, + "latitude": {"value": 20.43108005561, "unit": "deg"}, + "parallax": {"value": 0.4319, "unit": "mas"}, + "radialVelocity": {"value": None, "unit": "km/s"}, + "pmLongitude": {"value": -2.557, "unit": "mas/yr"}, + "pmLatitude": {"value": -5.482, "unit": "mas/yr"}, + "epoch": {"value": "2016.5"}, + "pmCosDeltApplied": {"value": True}, + "EpochPosition_errors": { + "dmrole": "errors", + "dmtype": "EpochPositionErrors", + "EpochPositionErrors_parallax": { + "dmrole": "parallax", + "dmtype": "PropertyError1D", + "sigma": {"value": 0.06909999996423721, "unit": "mas"}, + }, + "EpochPositionErrors_radialVelocity": { + "dmrole": "radialVelocity", + "dmtype": "PropertyError1D", + "sigma": {"value": None, "unit": "km/s"}, + }, + "EpochPositionErrors_position": { + "dmrole": "position", + "dmtype": "ErrorMatrix", + "sigma1": {"value": 0.0511, "unit": "mas"}, + "sigma2": {"value": 0.0477, "unit": "mas"}, + }, + "EpochPositionErrors_properMotion": { + "dmrole": "properMotion", + "dmtype": "ErrorMatrix", + "sigma1": {"value": 0.06400000303983688, "unit": "mas/yr"}, + "sigma2": {"value": 0.06700000166893005, "unit": "mas/yr"}, + }, + }, + "EpochPosition_correlations": { + "dmrole": "correlations", + "dmtype": "EpochPositionCorrelations", + "EpochPositionCorrelations_positionPm": { + "dmrole": "positionPm", + "dmtype": "Correlation22", + "isCovariance": {"value": True}, + "a2b1": {"value": -0.0085}, + "a2b2": {"value": -0.2983}, + "a1b1": {"value": -0.4109}, + "a1b2": {"value": -0.0072}, + }, + "EpochPositionCorrelations_parallaxPm": { + "dmrole": "parallaxPm", + "dmtype": "Correlation12", + "isCovariance": {"value": True}, + "a1b1": {"value": -0.2603}, + "a1b2": {"value": -0.0251}, + }, + "EpochPositionCorrelations_positionParallax": { + "dmrole": "positionParallax", + "dmtype": "Correlation21", + "isCovariance": {"value": True}, + "a2b1": {"value": 0.0069}, + "a1b1": {"value": 0.1337}, + }, + "EpochPositionCorrelations_positionPosition": { + "dmrole": "positionPosition", + "dmtype": "Correlation22", + "isCovariance": {"value": True}, + "a2b1": {"value": 0.1212}, + "a1b2": {"value": 0.1212}, + }, + "EpochPositionCorrelations_properMotionPm": { + "dmrole": "properMotionPm", + "dmtype": "Correlation22", + "isCovariance": {"value": True}, + "a2b1": {"value": 0.2688}, + "a1b2": {"value": 0.2688}, + }, + }, + "EpochPosition_coordSys": { + "dmid": "_spacesys_icrs", + "dmrole": "coordSys", + "dmtype": "SpaceSys", + "PhysicalCoordSys_frame": { + "dmrole": "frame", + "dmtype": "SpaceFrame", + "spaceRefFrame": {"value": "ICRS"}, + }, + }, + } + break + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_cone_search(vizier_url): + """ + Test that data returned by Vizier CS (1 row) and read through MIVOt are those expected + """ + scs_srv = SCSService(vizier_url) + m_viewer = MivotViewer( + scs_srv.search( + pos=SkyCoord(ra=52.26708 * u.degree, dec=59.94027 * u.degree, frame="icrs"), + radius=0.05, + ) + ) + mivot_instance = m_viewer.dm_instance + assert mivot_instance.dmtype == "EpochPosition" + assert mivot_instance.Coordinate_coordSys.spaceRefFrame.value == "ICRS" + ra = [] + dec = [] + pmra = [] + pmdec = [] + while m_viewer.next(): + ra.append(mivot_instance.longitude.value) + dec.append(mivot_instance.latitude.value) + pmra.append(mivot_instance.pmLongitude.value) + pmdec.append(mivot_instance.pmLatitude.value) + assert ra == [52.26722684] + assert dec == [59.94033461] + assert pmra == [-0.82] + assert pmdec == [-1.85] diff --git a/pyvo/mivot/tests/test_vizier_cs.py b/pyvo/mivot/tests/test_vizier_cs.py new file mode 100644 index 000000000..906429f52 --- /dev/null +++ b/pyvo/mivot/tests/test_vizier_cs.py @@ -0,0 +1,129 @@ +''' +The first service in operation the annotates query responses in the fly is Vizier +https://cds/viz-bin/mivotconesearch/VizierParams +Data are mapped o the EPochPropagtion model as it is implemented in the current code. +This test case is based on 2 VOTables: + +- The Vizier native (vizier_cs_withname.xml) where all ATTRIBUTE@ref are + based on FIELD@name even when a field has an ID. +- The patched vizier (vizier_cs_withid.xml) where all ATTRIBUTE@ref are + based on FIELD@ID or FIELD@name if it exists. +The test checks that: +- The position fields can be retrieved through the mapping. +- Both cases give the same results +A third test checks the case where a referenced column does not exist. + +Created on 26 janv. 2024 +@author: michel +''' +import os +import pytest +from urllib.request import urlretrieve +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot import MivotViewer +from pyvo.mivot.utils.exceptions import MivotException + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) + + +@pytest.fixture +def data_sample_url(): + return "https://raw.githubusercontent.com/ivoa/dm-usecases/main/pyvo-ci-sample/" + + +@pytest.fixture +def delt_coo(): + """ acceptable delta for coordinate value comparisons + """ + return 0.0000005 + + +@pytest.fixture +def path_to_withname(data_path, data_sample_url): + + votable_name = "vizier_cs_withname.xml" + votable_path = os.path.join(data_path, "data", votable_name) + urlretrieve(data_sample_url + votable_name, + votable_path) + + yield votable_path + os.remove(votable_path) + + +@pytest.fixture +def path_to_withid(data_path, data_sample_url): + + votable_name = "vizier_cs_withid.xml" + votable_path = os.path.join(data_path, "data", votable_name) + urlretrieve(data_sample_url + votable_name, + votable_path) + + yield votable_path + os.remove(votable_path) + + +@pytest.fixture +def path_to_badref(data_path, data_sample_url): + + votable_name = "vizier_cs_badref.xml" + votable_path = os.path.join(data_path, "data", votable_name) + urlretrieve(data_sample_url + votable_name, + votable_path) + + yield votable_path + os.remove(votable_path) + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_with_name(path_to_withname, delt_coo): + """ Test that the epoch propagation works with all FIELDs referenced by name or by ID + """ + m_viewer = MivotViewer(votable_path=path_to_withname) + m_viewer.next() + mivot_object = m_viewer.dm_instance + + assert abs(mivot_object.longitude.value - 52.2340018) < delt_coo + assert abs(mivot_object.latitude.value - 59.8937333) < delt_coo + assert abs(mivot_object.pmLongitude.value - 1.5) < delt_coo + assert abs(mivot_object.pmLatitude.value - -12.30000019) < delt_coo + assert str(mivot_object.epoch.value) == '2013.418' + assert str(mivot_object.Coordinate_coordSys.spaceRefFrame.value) == 'ICRS' + + m_viewer.next() + + assert abs(mivot_object.longitude.value - 32.2340018) < delt_coo + assert abs(mivot_object.latitude.value - 49.8937333) < delt_coo + assert abs(mivot_object.pmLongitude.value - 1.5) < delt_coo + assert abs(mivot_object.pmLatitude.value - -12.30000019) < delt_coo + assert str(mivot_object.epoch.value) == '2013.418' + assert str(mivot_object.Coordinate_coordSys.spaceRefFrame.value) == 'ICRS' + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_with_id(path_to_withid, delt_coo): + """ Test that the epoch propagation works with all FIELDs referenced by name or by ID + """ + # Test with all FILELDs referenced by names + m_viewer = MivotViewer(votable_path=path_to_withid) + m_viewer.next() + mivot_instance = m_viewer.dm_instance + assert abs(mivot_instance.longitude.value - 52.2340018) < delt_coo + assert abs(mivot_instance.latitude.value - 59.8937333) < delt_coo + assert abs(mivot_instance.pmLongitude.value - 1.5) < delt_coo + assert abs(mivot_instance.pmLatitude.value - -12.30000019) < delt_coo + assert str(mivot_instance.epoch.value) == '2013.418' + + +@pytest.mark.remote_data +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_bad_ref(path_to_badref, delt_coo): + """ Test that the epoch propagation works with all FIELDs referenced by name or by ID + """ + # Test with all FILELDs referenced by names + with (pytest.raises(MivotException, match="Attribute mango:EpochPosition.epoch can not be set.*")): + MivotViewer(votable_path=path_to_badref) diff --git a/pyvo/mivot/tests/test_xml_viewer.py b/pyvo/mivot/tests/test_xml_viewer.py new file mode 100644 index 000000000..e643ea2bc --- /dev/null +++ b/pyvo/mivot/tests/test_xml_viewer.py @@ -0,0 +1,72 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Test for mivot.viewer.model_viewer_level2.py +""" +import os +import pytest +try: + from defusedxml.ElementTree import Element as element +except ImportError: + from xml.etree.ElementTree import Element as element +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot import MivotViewer +from pyvo.mivot.utils.exceptions import MivotException + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_xml_viewer(m_viewer): + + m_viewer.next() + xml_viewer = m_viewer.xml_viewer + with pytest.raises(MivotException, + match="Cannot find dmrole wrong_role in any instances of the VOTable"): + xml_viewer.get_instance_by_role("wrong_role") + + with pytest.raises(MivotException, + match="Cannot find dmrole wrong_role in any instances of the VOTable"): + xml_viewer.get_instance_by_role("wrong_role", all_instances=True) + + with pytest.raises(MivotException, + match="Cannot find dmtype wrong_dmtype in any instances of the VOTable"): + xml_viewer.get_instance_by_type("wrong_dmtype") + + with pytest.raises(MivotException, + match="Cannot find dmtype wrong_dmtype in any instances of the VOTable"): + xml_viewer.get_instance_by_type("wrong_dmtype", all_instances=True) + + with pytest.raises(MivotException, + match="Cannot find dmrole wrong_role in any collections of the VOTable"): + xml_viewer.get_collection_by_role("wrong_role") + + with pytest.raises(MivotException, + match="Cannot find dmrole wrong_role in any collections of the VOTable"): + xml_viewer.get_collection_by_role("wrong_role", all_instances=True) + + instances_list_role = xml_viewer.get_instance_by_role("cube:MeasurementAxis.measure") + assert isinstance(instances_list_role, element) + + instances_list_role = xml_viewer.get_instance_by_role("cube:MeasurementAxis.measure", all_instances=True) + assert len(instances_list_role) == 3 + + instances_list_type = xml_viewer.get_instance_by_type("cube:Observable") + assert isinstance(instances_list_type, element) + + instances_list_type = xml_viewer.get_instance_by_type("cube:Observable", all_instances=True) + assert len(instances_list_type) == 3 + + collections_list_role = xml_viewer.get_collection_by_role("cube:NDPoint.observable") + assert isinstance(collections_list_role, element) + + collections_list_role = xml_viewer.get_collection_by_role("cube:NDPoint.observable", all_instances=True) + assert len(collections_list_role) == 1 + + +@pytest.fixture +def m_viewer(data_path): + return MivotViewer(os.path.join(data_path, "data", "test.mivot_viewer.xml"), + tableref="Results") + + +@pytest.fixture +def data_path(): + return os.path.dirname(os.path.realpath(__file__)) diff --git a/pyvo/mivot/utils/__init__.py b/pyvo/mivot/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyvo/mivot/utils/dict_utils.py b/pyvo/mivot/utils/dict_utils.py new file mode 100644 index 000000000..bd9d76e97 --- /dev/null +++ b/pyvo/mivot/utils/dict_utils.py @@ -0,0 +1,58 @@ +""" +Utility class to process dictionary. +""" +import json +import logging +from pyvo.mivot.utils.exceptions import MivotException +from pyvo.mivot.utils.json_encoder import MivotJsonEncoder + + +class DictUtils: + """ + Static class implementing convenient operations on dictionaries. + """ + + @staticmethod + def read_dict_from_file(filename, fatal=False): + """ + Read a dictionary from a file and raise an exception if something goes wrong. + Parameters: + - filename (str): The filename. Any file that can be processed by json.load is accepted + - fatal (bool): Triggers a system exit if True. + Returns: + - dict: The dictionary extracted from the file. + Raises: + - DataFormatException: If the file has an incorrect format. + """ + try: + logging.debug("Reading json from %s", filename) + from collections import OrderedDict + with open(filename, 'r') as file: + return json.load(file, object_pairs_hook=OrderedDict) + except Exception as exception: + if fatal: + raise MivotException("reading {}".format(filename)) + else: + logging.error("{} reading {}".format(exception, filename)) + + @staticmethod + def _get_pretty_json(dictionary): + """ + Return a pretty string representation of the dictionary. + Parameters: + - dictionary (dict): The dictionary. + Returns: + - str: A pretty string representation of the dictionary. + """ + return json.dumps(dictionary, + indent=2, + cls=MivotJsonEncoder) + + @staticmethod + def print_pretty_json(dictionary): + """ + Print out a pretty string representation of the dictionary. + Parameters: + - dictionary (dict): The dictionary. + """ + print(DictUtils._get_pretty_json(dictionary)) diff --git a/pyvo/mivot/utils/exceptions.py b/pyvo/mivot/utils/exceptions.py new file mode 100644 index 000000000..624e88680 --- /dev/null +++ b/pyvo/mivot/utils/exceptions.py @@ -0,0 +1,31 @@ +""" +MIVOT Exceptions + +3 exception classes +- AstropyVersionException that prevent to use the package +- MappingException if the annotation cannot be processed (e.g. no MIVOT block) + but the VOtable parsing can continue +- MivotException in any other case (block the processing) +""" + + +class MivotException(Exception): + """ + The annotation block is there but something went wrong with its processing + """ + + +class MappingException(Exception): + """ + Exception raised if a Resource or MIVOT element can't be mapped for one of these reasons: + - It doesn't match with any Resource/MIVOT element expected. + - It matches with too many Resource/MIVOT elements than expected. + This exception is trapped by the Viewer so that the processing can continue by ignoring + the annotations + """ + + +class AstropyVersionException(Exception): + """ + Exception raised if the version of astropy is not compatible with MIVOT. + """ diff --git a/pyvo/mivot/utils/json_encoder.py b/pyvo/mivot/utils/json_encoder.py new file mode 100644 index 000000000..37ccf7483 --- /dev/null +++ b/pyvo/mivot/utils/json_encoder.py @@ -0,0 +1,30 @@ +import json +import numpy + + +class MivotJsonEncoder(json.JSONEncoder): + """ + Custom JSON encoder for NumPy data types. + This class extends the default JSONEncoder to handle NumPy integers, + floating-point numbers, and arrays during JSON encoding. + """ + def default(self, obj): + """ + Serialize NumPy data types to their Python equivalents for JSON encoding. + Parameters + ---------- + obj : Any + The object to be encoded. + Returns + ------- + Any + The serialized version of the object. + """ + if isinstance(obj, numpy.integer): + return int(obj) + elif isinstance(obj, numpy.floating): + return float(obj) + elif isinstance(obj, numpy.ndarray): + return obj.tolist() + else: + return super(MivotJsonEncoder, self).default(obj) diff --git a/pyvo/mivot/utils/mivot_utils.py b/pyvo/mivot/utils/mivot_utils.py new file mode 100644 index 000000000..df6925f75 --- /dev/null +++ b/pyvo/mivot/utils/mivot_utils.py @@ -0,0 +1,120 @@ +''' +Some utilities making easier the transformation of Mivot elements into dictionary components. +These dictionaries are used to generate ``MivotInstance`` objects +''' +import numpy + + +class MivotUtils(object): + @staticmethod + def xml_to_dict(element): + """ + Recursively create a nested dictionary from the XML tree structure, preserving the hierarchy. + Each object in the dictionary is represented by a new dictionary with dmrole: {}. + The processing of elements depends on the tag: + - For INSTANCE, a new dictionary is created. + - For COLLECTION, a list is created. + - For ATTRIBUTE, a leaf is created in the tree structure with dmtype, dmrole, value, unit, and ref. + Parameters + ---------- + element (~`xml.etree.ElementTree.Element`) : The XML element to convert to a dictionary. + Returns + ------- + dict: The nested dictionary representing the XML tree structure. + """ + dict_result = {} + for key, value in element.attrib.items(): + dict_result[key] = value + for child in element: + dmrole = child.get("dmrole") + if child.tag == "ATTRIBUTE": + dict_result[dmrole] = MivotUtils.attribute_to_dict(child) + elif child.tag == "INSTANCE": # INSTANCE is recursively well managed by the function _to_dict + dict_result[dmrole] = MivotUtils.xml_to_dict(child) + elif child.tag == "COLLECTION": + dict_result[dmrole] = MivotUtils.collection_to_dict(child) + return dict_result + + @staticmethod + def attribute_to_dict(child): + """ + Convert an ATTRIBUTE element to a dictionary. + ATTRIBUTE is always a leaf, so it is not recursive. + Parameters + ---------- + child (~`xml.etree.ElementTree.Element`): ATTRIBUTE XML element to convert. + Returns + ------- + dict: A dictionary representing the ATTRIBUTE element with keys: + 'dmtype', 'dmrole', 'value', 'unit', and 'ref'. + """ + attribute = {} + if child.get('dmtype') is not None: + attribute['dmtype'] = child.get("dmtype") + if child.get("value") is not None: + attribute['value'] = MivotUtils.cast_type_value(child.get("value"), child.get("dmtype")) + else: + attribute['value'] = None + if child.get("unit") is not None: + attribute['unit'] = child.get("unit") + else: + attribute['unit'] = None + if child.get("ref") is not None: + attribute['ref'] = child.get("ref") + else: + attribute['ref'] = None + return attribute + + @staticmethod + def collection_to_dict(child): + """ + Convert a COLLECTION element to a list of dictionaries. + COLLECTION is always represented as a list, and each element of the COLLECTION is added to the list. + Parameters + ---------- + child (`~`xml.etree.ElementTree.Element``): COLLECTION XML element to convert. + Returns + ------- + list: list of dictionaries representing the elements of the COLLECTION. + """ + collection_items = [] + for child_coll in child: + collection_items.append(MivotUtils.xml_to_dict(child_coll)) + return collection_items + + @staticmethod + def cast_type_value(value, dmtype): + """ + Cast the value of an ATTRIBUTE based on its dmtype. + As the type of ATTRIBUTE values returned in the dictionary is string by default, + this function is used to cast them based on their dmtype. + Parameters + ---------- + value (str): value of the ATTRIBUTE. + dmtype (str): dmtype of the ATTRIBUTE. + Returns + ------- + Union[bool, float, str, None] + The cast value based on the dmtype. + """ + if type(value) is numpy.float32 or type(value) is numpy.float64: + return float(value) + lower_dmtype = dmtype.lower() + if isinstance(value, str): + lower_value = value.lower() + else: + lower_value = value + if "bool" in lower_dmtype: + if value == "1" or lower_value == "true" or lower_value: + return True + else: + return False + elif lower_value in ('notset', 'noset', 'null', 'none', 'nan') or value is None: + return None + elif (isinstance(value, numpy.ndarray) or isinstance(value, numpy.ma.core.MaskedConstant) + or value == '--'): + return None + elif "real" in lower_dmtype or "double" in lower_dmtype or "float" in lower_dmtype: + return float(value) + else: + return value diff --git a/pyvo/mivot/utils/vocabulary.py b/pyvo/mivot/utils/vocabulary.py new file mode 100644 index 000000000..17187cd7b --- /dev/null +++ b/pyvo/mivot/utils/vocabulary.py @@ -0,0 +1,73 @@ +""" +MIVOT vocabulary and regular expressions. +""" +import re +from astropy import units as u +from pyvo.utils import prototype_feature + + +class Constant: + """ + Class used to set constant to identify XML attributes added to the MIVOT ATTRIBUTES + """ + FIRST_TABLE = "first_table" + FIELD_UNIT = "field_unit" + COL_INDEX = "col_index" + ROOT_COLLECTION = "root_collection" + NOT_SET = "NotSet" + ANONYMOUS_TABLE = "AnonymousTable" + + +# Regexp pattern to check no valid mapping is present +NoMapping = re.compile(r".REPORT\s+status=['\"]KO") + +unit_mapping = { + "deg": u.degree, + "rad": u.radian, + "hourangle": u.hourangle, + "arcsec": u.arcsec, + "mas": u.mas, + "pc": u.pc, + "km": u.km, + "m": u.m, + "mas/yr": u.mas / u.yr, + "mas/y": u.mas / u.yr, + "km/s": u.km / u.s, +} + + +@prototype_feature('MIVOT') +class Ele: + """ + Constant used to identify MIVOT Element + """ + namespace = "" + VODML = namespace + "VODML" + MODEL = namespace + "MODEL" + GLOBALS = namespace + "GLOBALS" + TEMPLATES = namespace + "TEMPLATES" + INSTANCE = namespace + "INSTANCE" + ATTRIBUTE = namespace + "ATTRIBUTE" + COLLECTION = namespace + "COLLECTION" + JOIN = namespace + "JOIN" + REFERENCE = namespace + "REFERENCE" + WHERE = namespace + "WHERE" + NOROLE = "NOROLE" + + +@prototype_feature('MIVOT') +class Att: + """ + Constant used to identify attributes in MIVOT Element + """ + dmrole = "dmrole" + dmtype = "dmtype" + dmid = "dmid" + name = "name" + value = "value" + dmref = "dmref" + tableref = "tableref" + sourceref = "sourceref" + ref = "ref" + primarykey = "PRIMARY_KEY" + foreignkey = "foreignkey" diff --git a/pyvo/mivot/utils/xml_utils.py b/pyvo/mivot/utils/xml_utils.py new file mode 100644 index 000000000..7c3f40cc9 --- /dev/null +++ b/pyvo/mivot/utils/xml_utils.py @@ -0,0 +1,129 @@ +""" +Utility class to process XML. +""" +from pyvo.mivot.utils.xpath_utils import XPath +import xml.etree.ElementTree as ET +from pyvo.mivot.utils.vocabulary import Constant +from pyvo.mivot.utils.vocabulary import Att +from pyvo.mivot.utils.exceptions import MivotException + + +class XmlUtils: + """ + Static class implementing convenient operations on XML + """ + @staticmethod + def pretty_print(xmltree): + """ + Pretty print an XML tree. + Parameters + ---------- + xmltree (~`xml.etree.ElementTree.Element`): XML tree to pretty print. + """ + print(XmlUtils.pretty_string(xmltree)) + + @staticmethod + def pretty_string(xmltree): + """ + Return a pretty string representation of an XML tree. + Parameters + ---------- + xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string. + Returns + ------- + str: The pretty string representation of the XML tree. + """ + if hasattr(xmltree, 'getroot'): + XmlUtils.indent(xmltree.getroot()) + new_xml = ET.tostring(xmltree.getroot(), encoding='unicode') + else: + XmlUtils.indent(xmltree) + new_xml = ET.tostring(xmltree, encoding='unicode') + return new_xml.replace("ns0:", "") + + @staticmethod + def indent(elem, level=0): + """ + Indent an XML tree. + Parameters + ---------- + elem (~`xml.etree.ElementTree.Element`): XML tree to indent. + level (int): level of indentation. + Returns + ------- + ~`xml.etree.ElementTree.Element` + The indented XML tree. + """ + i = "\n" + level * " " + j = "\n" + (level - 1) * " " + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = i + " " + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for subelem in elem: + XmlUtils.indent(subelem, level + 1) + if not elem.tail or not elem.tail.strip(): + elem.tail = j + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = j + return elem + + @staticmethod + def add_column_indices(mapping_block, index_map): + """ + Add column ranks to attributes having a ref. + Using ranks allows identifying columns even when NumPy arrays have been serialized as []. + Parameters + ---------- + mapping_block : ~`xml.etree.ElementTree.Element` + The XML mapping block. + index_map : dict + A dictionary mapping ref values to column indices. + """ + for ele in XPath.x_path(mapping_block, ".//ATTRIBUTE"): + attr_ref = ele.get(Att.ref) + if attr_ref is not None and attr_ref != Constant.NOT_SET: + field_desc = None + if attr_ref in index_map: + field_desc = index_map[attr_ref] + else: + for _, value in index_map.items(): + if value["ID"] == attr_ref: + field_desc = value + break + if not field_desc: + if not ele.get(Att.value): + raise MivotException( + f"Attribute {ele.get(Att.dmrole)} can not be set:" + f" references a non existing column: {attr_ref} " + f"and has no default value") + else: + ele.attrib.pop(Att.ref, None) + if field_desc: + ele.attrib[Constant.COL_INDEX] = str(field_desc["indx"]) + if field_desc["ID"] != attr_ref: + ele.set(Att.ref, field_desc["ID"]) + + @staticmethod + def add_column_units(mapping_block, unit_map): + """ + Add field units to attributes having a ref. + Used for performing unit conversions. + Parameters + ---------- + mapping_block : ~`xml.etree.ElementTree.Element` + The XML mapping block. + unit_map : dict + A dictionary mapping ref values to units. + """ + for ele in XPath.x_path(mapping_block, ".//ATTRIBUTE"): + ref = ele.get(Att.ref) + if ref is not None and ref != Constant.NOT_SET: + unit = None + if ref in unit_map: + unit = unit_map[ref].__str__() + else: + unit = "" + ele.attrib[Constant.FIELD_UNIT] = unit diff --git a/pyvo/mivot/utils/xpath_utils.py b/pyvo/mivot/utils/xpath_utils.py new file mode 100644 index 000000000..bbe894e1c --- /dev/null +++ b/pyvo/mivot/utils/xpath_utils.py @@ -0,0 +1,107 @@ +""" +Utility class performing XPath queries on XML trees. +""" + + +class XPath: + """ + Static class use to perform XPath queries on XML trees. + """ + @staticmethod + def x_path(etree, path): + """ + Return all the elements of the XML tree that match the given XPath query. + Parameters + ---------- + etree : ~`xml.etree.ElementTree.Element` + The XML tree to query. + path : str + The XPath query to perform. + Returns + ------- + list + The list of all the elements of the XML tree that match the given XPath query. + """ + return etree.findall(path) + + @staticmethod + def x_path_contains(etree, path, key, value): + """ + Return all the elements of the XML tree that match the given + XPath query with a given attribute containing a given value. + Example of a path: ".//INSTANCE[contains(@dmtype,'dmtype_pattern')]" + Parameters + ---------- + etree : ~`xml.etree.ElementTree.Element` + The XML tree to query. + path : str + The XPath query to perform. + key : str + The attribute to look for. + value : str + The value to look for. + Returns + ------- + list + The list of all the elements of the XML tree that match the + given XPath query with a given attribute containing a given value. + """ + result = [] + xset = etree.findall(path) + for ele in xset: + if value in ele.get(key): + result.append(ele) + return result + + @staticmethod + def x_path_startwith(etree, path): + """ + Return all the elements of the XML tree that match the given + XPath query with a given Tag starting with a given value. + Example of a path: ".//*[starts-with(name(), 'REFERENCE_')]" + This function is only used in the static reference resolver to find all the REFERENCEs and JOINs. + It adds a counter to the path to find all the REFERENCEs and JOINs. + Parameters + ---------- + etree : `xml.etree.ElementTree.Element` + The XML tree to query. + path : str + The XPath query to perform. + Returns + ------- + list + The list of all the elements of the XML tree that match the given XPath query. + """ + cnt = 1 + result = [] + run = True + while run: + if etree.find(path + str(cnt)) is not None: + result.append(etree.find(path + str(cnt))) + cnt += 1 + else: + run = False + return result + + @staticmethod + def select_elements_by_atttribute(etree, element, attribute, attribute_value): + """ + Select all xml elements named 'element' having @attribute=attribute_value + Parameters + ---------- + etree : `xml.etree.ElementTree.Element` + The XML tree to query. + element: str + name of the searched elements + attribute: str + name of the element attribute used for the selection + attribute_value: str + attribute value of the selected elements + Returns + ------- + list + The list of all the elements of the XML tree that match the given XPath query. + """ + return XPath.x_path(etree, + f'.//{element}[@{attribute}="{attribute_value}"]' + ) diff --git a/pyvo/mivot/version_checker.py b/pyvo/mivot/version_checker.py new file mode 100644 index 000000000..43905772d --- /dev/null +++ b/pyvo/mivot/version_checker.py @@ -0,0 +1,14 @@ +from astropy import version as astropy_version + + +def check_astropy_version(): + """ + Check if the installed version of astropy is compatible with MIVOT. + """ + if not astropy_version.version: + return False + if astropy_version.version < "6.0": + print(f"Astropy version {astropy_version.version} is below " + f"the required version 6.0 for the use of MIVOT.") + return False + return True diff --git a/pyvo/mivot/viewer/__init__.py b/pyvo/mivot/viewer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyvo/mivot/viewer/mivot_instance.py b/pyvo/mivot/viewer/mivot_instance.py new file mode 100644 index 000000000..c90bec8d2 --- /dev/null +++ b/pyvo/mivot/viewer/mivot_instance.py @@ -0,0 +1,219 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +MivotInstance is the root of the Python generated classes. +Instances of MivotInstance are built from a dictionary issued +from the XML view of the mapped model. +This dictionary is used to extend the object with all components +(classes, attributes, collections) necessary to reproduce the structure +of the mapped model. +Instances of this class are built by `pyvo.mivot.viewer.mivot_viewer`. +Although attribute values can be changed by users, this class is first +meant to provide a convenient access the mapped VOTable data +""" +from astropy import time +from pyvo.mivot.utils.vocabulary import unit_mapping +from pyvo.utils.prototype import prototype_feature +from pyvo.mivot.utils.mivot_utils import MivotUtils +from pyvo.mivot.utils.dict_utils import DictUtils + +# list of model leaf parameters that must be hidden for the final user +hk_parameters = ["astropy_unit", "ref"] + + +@prototype_feature('MIVOT') +class MivotInstance: + """ + MivotInstance holds the dictionary (__dict__) similar with the mapped model structure + where the references have been resolved. + The dictionary keeps the hierarchy of the XML : + "key" : {not a leaf} means key is the dmtype of an INSTANCE + "key" : {leaf} means key is the dmrole of an ATTRIBUTE + "key" : "value" means key is an element of ATTRIBUTE + "key" : [] means key is the dmtype of a COLLECTION + """ + def __init__(self, **instance_dict): + """ + Constructor of the MIVOT class. + + Parameters + ---------- + kwargs (dict): Dictionary of the XML object. + """ + self._create_class(**instance_dict) + + def __repr__(self): + """ + return a human readable (json) representation of object + """ + return DictUtils._get_pretty_json(self.dict) + + @property + def hk_dict(self): + """ + return a human readable (dict) representation of object with a few + housekeeping data such as column references. This might be used + to apply the mapping out of the MivotViewer context + """ + return self._get_class_dict(self) + + @property + def dict(self): + """ + return a human readable (dict) representation of object + """ + return self._get_class_dict(self, slim=True) + + def _create_class(self, **kwargs): + """ + Recursively initialize the MIVOT class with the dictionary of the XML object got in MivotViewer. + For the unit of the ATTRIBUTE, we add the Astropy unit or the Astropy time equivalence by comparing + the value of the unit with values in time.TIME_FORMATS.keys() which is the list of time formats. + We do the same with the unit_mapping dictionary, which is the list of Astropy units. + + Parameters + ---------- + kwargs (dict): Dictionary of the XML object. + """ + for key, value in kwargs.items(): + if isinstance(value, list): # COLLECTION + setattr(self, self._remove_model_name(key), []) + for item in value: + getattr(self, self._remove_model_name(key)).append(MivotInstance(**item)) + elif isinstance(value, dict): # INSTANCE + if not self._is_leaf(**value): + setattr(self, self._remove_model_name(key, role_instance=True), MivotInstance(**value)) + if self._is_leaf(**value): + setattr(self, self._remove_model_name(key), MivotInstance(**value)) + else: # ATTRIBUTE + if key == 'value': # We cast the value read in the row + setattr(self, self._remove_model_name(key), + MivotUtils.cast_type_value(value, getattr(self, 'dmtype'))) + else: + setattr(self, self._remove_model_name(key), self._remove_model_name(value)) + if key == 'unit': # We convert the unit to astropy unit or to astropy time format if possible + # The first Vizier implementation used mas/year for the mapped pm unit: let's correct it + value = value.replace("year", "yr") if value else None + if value in unit_mapping.keys(): + setattr(self, "astropy_unit", unit_mapping[value]) + elif value in time.TIME_FORMATS.keys(): + setattr(self, "astropy_unit_time", value) + + def update(self, row, ref=None): + """ + Update the MIVOT class with the new data row. + For each leaf of the MIVOT class, we update the value with the new data row. + + Parameters + ---------- + row (astropy.table.row.Row): The new data row. + ref (str, optional):The reference of the data row, default is None. + """ + for key, value in vars(self).items(): + if isinstance(value, list): + for item in value: + item.update(row=row) + elif isinstance(value, MivotInstance): + if isinstance(vars(value), dict): + if 'value' not in vars(value): + value.update(row=row) + if 'value' in vars(value): + value.update(row=row, ref=getattr(value, 'ref')) + else: + if key == 'value' and ref is not None and ref != 'null': + setattr(self, self._remove_model_name(key), + MivotUtils.cast_type_value(row[ref], getattr(self, 'dmtype'))) + + @staticmethod + def _remove_model_name(value, role_instance=False): + """ + Remove the model name before each colon ":" as well as the type of the object before each point ".". + If it is an INSTANCE of INSTANCEs, the dmrole represented as the key needs to keep his type object. + In this case (`role_instance=True`), we just replace the point "." With an underscore "_". + - if role_instance: a:b.c -> b_c else c + + Parameters + ---------- + value (str): The string to process. + role_instance (bool, optional): If True, keeps the type object for dmroles representing + an INSTANCE of INSTANCEs. Default is False. + """ + if isinstance(value, str): + # We first find the model_name before the colon + index_underscore = value.find(":") + if index_underscore != -1: + # Then we find the object type before the point + next_index_underscore = value.rfind(".") + if next_index_underscore != -1 and not role_instance: + value_after_underscore = value[next_index_underscore + 1:] + else: + value_after_underscore = (value[index_underscore + 1:] + .replace(':', '_').replace('.', '_')) + return value_after_underscore + return value # Returns unmodified string if "_" wasn't found + else: + return value + + def _is_leaf(self, **kwargs): + """ + Check if the dictionary is an ATTRIBUTE. + + Parameters + ---------- + **kwargs (dict): The dictionary to check. + Returns + ------- + bool: True if the dictionary is an ATTRIBUTE, False otherwise. + """ + if isinstance(kwargs, dict): + for _, value in kwargs.items(): + if isinstance(value, dict): + return False + return True + + def _get_class_dict(self, obj, classkey=None, slim=False): + """ + Recursively displays a serializable dictionary. + This function is only used for debugging purposes. + + Parameters + ---------- + obj (dict or object): The dictionary or object to display. + classkey (str, optional): The key to use for the object's class name + in the dictionary, default is None. + slim (bool, optional): if true, only @values and @units (if not empty) are + attached to model leaves. + @dmtype and @ref attributes are ignored + Returns + ------- + dict or object + The serializable dictionary representation of the input. + """ + if isinstance(obj, dict): + data = {} + for (k, v) in obj.items(): + data[k] = self._get_class_dict(v, classkey, slim=slim) + return data + elif hasattr(obj, "_ast"): + return self._get_class_dict(obj._ast()) + elif hasattr(obj, "__iter__") and not isinstance(obj, str): + return [self._get_class_dict(v, classkey, slim=slim) for v in obj] + elif hasattr(obj, "__dict__"): + data = dict([(key, self._get_class_dict(value, classkey, slim=slim)) + for key, value in obj.__dict__.items() + if not callable(value) and not key.startswith('_')]) + # remove the house keeping parameters + if slim is True: + # data is atomic value (e.g. float): the type be hidden + if "ref" in data or "value" in data: + data.pop("dmtype", None) + # remove unit when not set + if "unit" in data and not data["unit"]: + data.pop("unit", None) + for hk_parameter in hk_parameters: + data.pop(hk_parameter, None) + + if classkey is not None and hasattr(obj, "__class__"): + data[classkey] = obj.__class__.__name__ + return data + else: + return obj diff --git a/pyvo/mivot/viewer/mivot_viewer.py b/pyvo/mivot/viewer/mivot_viewer.py new file mode 100644 index 000000000..a6f1224f7 --- /dev/null +++ b/pyvo/mivot/viewer/mivot_viewer.py @@ -0,0 +1,489 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +MivotViewer implements the user API for accessing mapped data. +- extracts the annotation block from the VOTable +- builds an XML view of the mapped model +- updates the leaves of the XML view with the values read in the data rows +- builds Python instances providing access to the mapped values by object attributes. + +The code below shows a typical use of `MivotViewer + + .. code-block:: python + + with MivotViewer(path_to_votable) as mivot_viewer: + print(f"mapped class id {mivot_instance.dmtype}") + print(f"space frame is {mivot_instance.Coordinate_coordSys.spaceRefFrame.value}") + + mivot_object = mivot_viewer.dm_instance + while mivot_viewer.next(): + print(f"latitude={mivot_object.latitude.value}") + print(f"longitude={mivot_object.longitude.value}") + +See `tests/test_user_api.py`to get different examples of the API usage. +""" +import logging +from copy import deepcopy +from astropy import version +from astropy.io.votable import parse +from astropy.io.votable.tree import VOTableFile +from pyvo.dal import DALResults +from pyvo.mivot.utils.vocabulary import Ele, Att +from pyvo.mivot.utils.vocabulary import Constant, NoMapping +from pyvo.mivot.utils.exceptions import (MappingException, + MivotException, + AstropyVersionException) +from pyvo.mivot.utils.xml_utils import XmlUtils +from pyvo.mivot.utils.xpath_utils import XPath +from pyvo.mivot.seekers.annotation_seeker import AnnotationSeeker +from pyvo.mivot.seekers.resource_seeker import ResourceSeeker +from pyvo.mivot.seekers.table_iterator import TableIterator +from pyvo.mivot.features.static_reference_resolver import StaticReferenceResolver +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot.viewer.mivot_instance import MivotInstance +from pyvo.utils.prototype import prototype_feature +from pyvo.mivot.utils.mivot_utils import MivotUtils +from pyvo.mivot.viewer.xml_viewer import XMLViewer +# Use defusedxml only if already present in order to avoid a new depency. +try: + from defusedxml import ElementTree as etree +except ImportError: + from xml.etree import ElementTree as etree + + +@prototype_feature('MIVOT') +class MivotViewer: + """ + MivotViewer is a PyVO table wrapper aiming at providing + a model view on VOTable data read with usual tools. + """ + def __init__(self, votable_path, tableref=None): + """ + Constructor of the MivotViewer class. + + Parameters + ---------- + votable_path : str, parsed VOTable or DALResults instance + VOTable that will be parsed with the parser of Astropy, + which extracts the annotation block. + tableref : str, optional + Used to identify the table to process. If not specified, + the first table is taken by default. + """ + if not check_astropy_version(): + raise AstropyVersionException(f"Astropy version {version.version} " + f"is below the required version 6.0 for the use of MIVOT.") + + if isinstance(votable_path, DALResults): + self._parsed_votable = votable_path.votable + elif isinstance(votable_path, VOTableFile): + self._parsed_votable = votable_path + else: + self._parsed_votable = parse(votable_path) + self._table_iterator = None + self._connected_table = None + self._connected_tableref = None + self._current_data_row = None + # when the search object is in GLOBALS + self._globals_instance = None + self._last_row = None + self._templates = None + self._resource = None + self._annotation_seeker = None + self._mapping_block = None + self._mapped_tables = [] + self._resource_seeker = None + self._dm_instance = None + try: + self._set_resource() + self._set_mapping_block() + self._resource_seeker = ResourceSeeker(self._resource) + self._set_mapped_tables() + self._connect_table(tableref) + self._init_instance() + except MappingException as mnf: + logging.error(str(mnf)) + + def __enter__(self): + """ with statement implementation """ + return self + + def __exit__(self, exc_type, exc_value, traceback): + """ with statement implementation """ + logging.info("MivotViewer closing..") + + def close(self): + """ with statement implementation """ + logging.info("MivotViewer is closed") + + @property + def votable(self): + """ + returns the Astropy parsed votable + """ + return self._parsed_votable + + @property + def annotation_seeker(self): + """ + Return an API to search various components in the XML mapping block. + """ + return self._annotation_seeker + + @property + def resource_seeker(self): + """ + Return an API to search various components in the VOTabel resource. + """ + return self._resource_seeker + + @property + def connected_table(self): + """ + getter for the identifier the astropy.table + instance the viewer is connected to + """ + return self._connected_table + + @property + def connected_table_ref(self): + """ getter for the identifier the table the viewer is connected to """ + return self._connected_tableref + + @property + def dm_instance(self): + """ + returns + ------- + A Python object (MivotInstance) built from the XML view of + the mapped model with attribute values set from the last values + of the last read data rows + """ + return self._dm_instance + + @property + def xml_view(self): + """ + returns + ------- + The XML view on the current data row + """ + return self.xml_viewer.view + + @property + def xml_viewer(self): + """ + returns + XMLViewer tuned to browse the TEMPLATES content + """ + # build a first XMLViewer for extract the content of the TEMPLATES element + model_view = XMLViewer(self._get_model_view()) + first_instance_dmype = self.get_first_instance_dmtype(tableref=self.connected_table_ref) + model_view.get_instance_by_type(first_instance_dmype) + + # return an XMLViewer tuned to process the TEMPLATES content + return XMLViewer(model_view._xml_view) + + @property + def table_row(self): + """ getter for the current astropy.table.array row """ + return self._current_data_row + + def next(self): + """ + jump to the next table row and update the MivotInstance instance + + returns + ------- + MivotInstance: the updated instance or None + it he able end has been reached + """ + self.next_table_row() + + if self._current_data_row is None: + return None + + if self._dm_instance is None: + xml_instance = self.xml_viewer.view + self._dm_instance = MivotInstance(**MivotUtils.xml_to_dict(xml_instance)) + self._dm_instance.update(self._current_data_row) + return self._dm_instance + + def get_table_ids(self): + """ + Return a list of the table located just below self._resource. + """ + if self.resource_seeker is None: + return None + return self.resource_seeker.get_table_ids() + + def get_globals_models(self): + """ + Get collection types in GLOBALS. + Collection types are GLOBALS/COLLECTION/INSTANCE@dmtype: + used for collections of static objects. + + Returns + ------- + dict + A dictionary containing the dmtypes of all the top-level INSTANCE/COLLECTION of GLOBALS. + The structure of the dictionary is {'COLLECTION': [dmtypes], 'INSTANCE': [dmtypes]}. + """ + if self._annotation_seeker is None: + return None + globals_models = {} + globals_models[Ele.COLLECTION] = self._annotation_seeker.get_globals_collection_dmtypes() + globals_models[Ele.INSTANCE] = self._annotation_seeker.get_globals_instance_dmtypes() + return globals_models + + def get_models(self): + """ + Get a dictionary of models and their URLs. + + Returns + ------- + dict: Model names and a lists of their URLs. + The format is {'model': [url], ...}. + """ + if self._annotation_seeker is None: + return None + return self._annotation_seeker.get_models() + + def get_templates_models(self): + """ + Get dmtypes (except ivoa:..) of all INSTANCE/COLLECTION of all TEMPLATES. + Note: COLLECTION not implemented yet. + + Returns + ------- + dict: A dictionary containing dmtypes of all INSTANCE/COLLECTION of all TEMPLATES. + The format is {'tableref': {'COLLECTIONS': [dmtypes], 'INSTANCE': [dmtypes]}, ...}. + """ + if self._annotation_seeker is None: + return None + templates_models = {} + gni = self._annotation_seeker.get_instance_dmtypes()[Ele.TEMPLATES] + for tid, tmplids in gni.items(): + templates_models[tid] = {Ele.COLLECTION: [], Ele.INSTANCE: tmplids} + return templates_models + + def next_table_row(self): + """ + Iterate once on the table row + + Returns: + numpy row: the current table row of None if the end of the table has been reached + """ + if self._table_iterator is None: + return None + self._current_data_row = self._table_iterator.get_next_row() + return self._current_data_row + + def rewind(self): + """ + Rewind the table iterator on the table the veizer is connected with. + """ + if self._table_iterator: + self._table_iterator.rewind() + + def get_first_instance_dmtype(self, tableref=None): + """ + Return the dmtype of the head INSTANCE (first TEMPLATES child). + If no INSTANCE is found, take the first COLLECTION. + + Parameters + ---------- + tableref : str or None, optional + Identifier of the table. + Returns + ------- + ~`xml.etree.ElementTree.Element` + The first child of TEMPLATES. + """ + if self._annotation_seeker is None: + return None + child_template = self._annotation_seeker.get_templates_block(tableref) + child = child_template.findall("*") + collection = XPath.x_path(self._annotation_seeker.get_templates_block(tableref), + ".//" + Ele.COLLECTION) + instance = XPath.x_path(self._annotation_seeker.get_templates_block(tableref), ".//" + Ele.INSTANCE) + if len(collection) >= 1: + collection[0].set(Att.dmtype, Constant.ROOT_COLLECTION) + (self._annotation_seeker.get_templates_block(tableref).find(".//" + Ele.COLLECTION) + .set(Att.dmtype, Constant.ROOT_COLLECTION)) + if len(child) > 1: + if len(instance) >= 1: + for inst in instance: + if inst in child: + return inst.get(Att.dmtype) + elif len(collection) >= 1: + for coll in collection: + if coll in child: + return coll.get(Att.dmtype) + elif len(child) == 1: + if child[0] in instance: + return child[0].get(Att.dmtype) + elif child[0] in collection: + return collection[0].get(Att.dmtype) + else: + raise MivotException("Can't find the first " + Ele.INSTANCE + + "/" + Ele.COLLECTION + " in " + Ele.TEMPLATES) + + def _connect_table(self, tableref=None): + """ + Iterate over the table identified by tableref. + Required to browse table data. + Connect to the first table if tableref is None. + + Parameters + ---------- + tableref : str or None, optional + Identifier of the table. If None, connects to the first table. + """ + if not self._resource_seeker: + raise MappingException("No mapping block found") + + stableref = tableref + if tableref is None: + stableref = "" + self._connected_tableref = Constant.FIRST_TABLE + logging.debug("Since " + Ele.TEMPLATES + "@table_ref is None, " + "the mapping will be applied to the first table." + ) + elif tableref not in self._mapped_tables: + raise MappingException(f"The table {self._connected_tableref} doesn't match with any " + f"mapped_table ({self._mapped_tables}) encountered in " + + Ele.TEMPLATES + ) + else: + self._connected_tableref = tableref + + self._connected_table = self._resource_seeker.get_table(tableref) + if self.connected_table is None: + raise MivotException(f"Cannot find table {stableref} in VOTable") + logging.debug("table %s found in VOTable", stableref) + self._templates = deepcopy(self.annotation_seeker.get_templates_block(tableref)) + if self._templates is None: + raise MivotException("Cannot find " + Ele.TEMPLATES + f" {stableref} ") + logging.debug(Ele.TEMPLATES + " %s found ", stableref) + self._table_iterator = TableIterator(self._connected_tableref, + self.connected_table.to_table()) + self._squash_join_and_references() + self._set_column_indices() + self._set_column_units() + + def _get_model_view(self, resolve_ref=True): + """ + Return an XML model view of the last read row. + This function resolves references by default. + + Parameters + ---------- + resolve_ref : bool, optional + If True, resolves the references. Default is True. + """ + templates_copy = deepcopy(self._templates) + if resolve_ref is True: + while StaticReferenceResolver.resolve(self._annotation_seeker, self._connected_tableref, + templates_copy) > 0: + pass + # Make sure the instances of the resolved references + # have both indexes and unit attribute + XmlUtils.add_column_indices(templates_copy, + self._resource_seeker + .get_id_index_mapping(self._connected_tableref)) + XmlUtils.add_column_units(templates_copy, + self._resource_seeker + .get_id_unit_mapping(self._connected_tableref)) + # for ele in templates_copy.xpath("//ATTRIBUTE"): + for ele in XPath.x_path(templates_copy, ".//ATTRIBUTE"): + ref = ele.get(Att.ref) + if ref is not None and ref != Constant.NOT_SET and Constant.COL_INDEX in ele.attrib: + index = ele.attrib[Constant.COL_INDEX] + ele.attrib[Att.value] = str(self._current_data_row[int(index)]) + return templates_copy + + def _init_instance(self): + """ + Read the first table row and build the MivotInstance (_instance attribute) from it. + The table row iterator in rewind at he end to make sure we won't lost the first data row. + """ + if self._dm_instance is None: + self.next_table_row() + first_instance = self.get_first_instance_dmtype(tableref=self.connected_table_ref) + xml_instance = self.xml_viewer.get_instance_by_type(first_instance) + self._dm_instance = MivotInstance(**MivotUtils.xml_to_dict(xml_instance)) + self.rewind() + return self._dm_instance + + def _set_mapped_tables(self): + """ + Set the mapped tables with a list of the TEMPLATES tablerefs. + """ + if not self.resource_seeker: + self._mapped_table = [] + else: + self._mapped_tables = self._annotation_seeker.get_templates() + + def _set_resource(self): + """ + select the first resource with @type=results + The annotations, if there are, are supposed to be there. + The case of multiple 'results' annotated is not taken into account yest + """ + + if len(self._parsed_votable.resources) < 1: + raise MivotException("No resource detected in the VOTable") + rnb = 0 + for res in self._parsed_votable.resources: + if res.type.lower() == "results": + logging.info("Resource %s selected", rnb) + self._resource = self._parsed_votable.resources[rnb] + return + rnb += 1 + raise MivotException("No resource @type='results'detected in the VOTable") + + def _set_mapping_block(self): + """ + Set the mapping block found in the resource and set the annotation_seeker + """ + if NoMapping.search(self._resource.mivot_block.content): + raise MappingException("Mivot block is not found") + # The namespace should be removed + self._mapping_block = ( + etree.fromstring(self._resource.mivot_block.content + .replace('xmlns="http://www.ivoa.net/xml/mivot"', '') + .replace("xmlns='http://www.ivoa.net/xml/mivot'", ''))) + self._annotation_seeker = AnnotationSeeker(self._mapping_block) + logging.info("Mapping block found") + + def _squash_join_and_references(self): + """ + Remove both JOINs and REFERENCEs from the templates + and store them in to be resolved later on. + This prevents the model view of being polluted with elements that are not in the model + """ + for ele in XPath.x_path_startwith(self._templates, ".//REFERENCE_"): + if ele.get("sourceref") is not None: + self._dyn_references = {ele.tag: deepcopy(ele)} + for child in list(ele): + ele.remove(child) + for ele in XPath.x_path_startwith(self._templates, ".//JOIN_"): + self._joins = {ele.tag: deepcopy(ele)} + for child in list(ele): + ele.remove(child) + + def _set_column_indices(self): + """ + Add column ranks to attribute having a ref. + Using ranks allow identifying columns even numpy raw have been serialised as [] + """ + index_map = self._resource_seeker.get_id_index_mapping(self._connected_tableref) + XmlUtils.add_column_indices(self._templates, index_map) + + def _set_column_units(self): + """ + Add field unit to attribute having a ref. + Used for performing unit conversions + """ + unit_map = self._resource_seeker.get_id_unit_mapping(self._connected_tableref) + XmlUtils.add_column_units(self._templates, unit_map) diff --git a/pyvo/mivot/viewer/xml_viewer.py b/pyvo/mivot/viewer/xml_viewer.py new file mode 100644 index 000000000..75103a0ec --- /dev/null +++ b/pyvo/mivot/viewer/xml_viewer.py @@ -0,0 +1,145 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +XMLViewer provides several getters on XML instances built by + `pyvo.mivot.viewer.mivot_viewer`. +""" +from pyvo.mivot.utils.exceptions import MivotException +from pyvo.mivot.utils.xpath_utils import XPath +from pyvo.utils.prototype import prototype_feature + + +@prototype_feature('MIVOT') +class XMLViewer: + """ + The XMLViewer is used by `~pyvo.mivot.viewer.mivot_viewer` + to extract from the XML serialization of the model, + elements that will be used to build the dictionary from which + the Python class holding the mapped model will be generated. + """ + def __init__(self, xml_view): + self._xml_view = xml_view + + @property + def view(self): + """ + getter returning the XML model view + + returns + ------- + XML model view to be parsed + by different methods + """ + return self._xml_view + + def get_instance_by_role(self, dmrole, all_instances=False): + """ + If all_instances is False, return the first INSTANCE matching with @dmrole. + If all_instances is True, return a list of all instances matching with @dmrole. + Parameters + ---------- + dmrole : str + The @dmrole to look for. + all_instances : bool, optional + If True, returns a list of all instances, otherwise returns the first instance. + Default is False. + Returns + ------- + Union[`xml.etree.ElementTree.Element`, List[`xml.etree.ElementTree.Element`], None] + If all_instances is False, returns the instance matching with @dmrole. + If all_instances is True, returns a list of all instances matching with @dmrole. + If no matching instance is found, returns None. + Raises + ------ + MivotElementNotFound + If dmrole is not found. + """ + instances = XPath.select_elements_by_atttribute( + self._xml_view, + "INSTANCE", + "dmrole", + dmrole) + + if len(instances) == 0: + raise MivotException( + f"Cannot find dmrole {dmrole} in any instances of the VOTable") + + if all_instances is False: + return instances[0] + else: + return instances + + def get_instance_by_type(self, dmtype, all_instances=False): + """ + Return the instance matching with @dmtype. + If all_instances is False, returns the first INSTANCE matching with @dmtype. + If all_instances is True, returns a list of all instances matching with @dmtype. + Parameters + ---------- + dmtype : str + The @dmtype to look for. + all : bool, optional + If True, returns a list of all instances, otherwise returns the first instance. + Default is False. + Returns + ------- + Union[~`xml.etree.ElementTree.Element`, List[~`xml.etree.ElementTree.Element`], None] + If all_instances is False, returns the instance matching with @dmtype. + If all_instances is True, returns a list of all instances matching with @dmtype. + If no matching instance is found, returns None. + Raises + ------ + MivotElementNotFound + If dmtype is not found. + """ + instances = XPath.select_elements_by_atttribute( + self._xml_view, + "INSTANCE", + "dmtype", + dmtype) + + if len(instances) == 0: + raise MivotException( + f"Cannot find dmtype {dmtype} in any instances of the VOTable") + + if all_instances is False: + return instances[0] + else: + return instances + + def get_collection_by_role(self, dmrole, all_instances=False): + """ + Return the collection matching with @dmrole. + If all_instances is False, returns the first COLLECTION matching with @dmrole. + If all_instances is True, returns a list of all COLLECTION matching with @dmrole. + Parameters + ---------- + dmrole : str + The @dmrole to look for. + all_instances : bool, optional + If True, returns a list of all COLLECTION, otherwise returns the first COLLECTION. + Default is False. + Returns + ------- + Union[~`xml.etree.ElementTree.Element`, List[~`xml.etree.ElementTree.Element`], None] + If all_instances is False, returns the collection matching with @dmrole. + If all_instances is True, returns a list of all collections matching with @dmrole. + If no matching collection is found, returns None. + Raises + ------ + MivotElementNotFound + If dmrole is not found. + """ + collections = XPath.select_elements_by_atttribute( + self._xml_view, + "COLLECTION", + "dmrole", + dmrole) + + if len(collections) == 0: + raise MivotException( + f"Cannot find dmrole {dmrole} in any collections of the VOTable") + + if all_instances is False: + return collections[0] + else: + return collections diff --git a/pyvo/utils/prototype.py b/pyvo/utils/prototype.py index 9dbc83606..659789d7e 100644 --- a/pyvo/utils/prototype.py +++ b/pyvo/utils/prototype.py @@ -8,11 +8,13 @@ __all__ = ['features', 'prototype_feature', 'activate_features', 'PrototypeWarning', 'PrototypeError'] - features: Dict[str, "Feature"] = { 'cadc-tb-upload': Feature('cadc-tb-upload', 'https://wiki.ivoa.net/twiki/bin/view/IVOA/TAP-1_1-Next', - False) + False), + 'MIVOT': Feature('MIVOT', + 'https://ivoa.net/documents/MIVOT/20230620/REC-mivot-1.0.pdf', + False) } @@ -104,7 +106,6 @@ def _parse_args(*args): def _make_decorator(feature_name): - def decorator(decorated): if inspect.isfunction(decorated): return _make_wrapper(feature_name, decorated) diff --git a/setup.cfg b/setup.cfg index 744472c3a..dc8eea95d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -74,6 +74,7 @@ test = pytest-doctestplus>=0.13 pytest-astropy requests-mock + defusedxml docs = sphinx-astropy @@ -82,6 +83,7 @@ pyvo.auth.tests = data/tap/*.xml pyvo.io.uws.tests = data/*.xml pyvo.io.vosi.tests = data/*.xml, data/tables/*.xml, data/capabilities/*.xml pyvo.registry.tests = data/*.xml, data/*.desise +pyvo.mivot.tests = data/*.xml, data/input/*.xml, data/output/*.xml pyvo.dal.tests = data/*.xml, data/*/* [coverage:run]