diff --git a/CHANGES.rst b/CHANGES.rst index db02e8afc..dbcf0371f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,9 @@ Enhancements and Fixes ---------------------- +- Extending the MIVOT module with the ability to build annotations component by component + and put them into a VOTable. [#627] + - Make deletion of TAP jobs optional via a new ``delete`` kwarg. [#640] Deprecations and Removals @@ -54,18 +57,20 @@ Enhancements and Fixes - MIVOT module: the model references in the dictionaries that are used to build ``MivotInstance`` objects are made more consistent [#551] -- RegTAP constraints involving tables other than rr.resource are now - done via subqueries for less duplication of interfaces. [#562, #572] - - MIVOT module: If the MIVOT annotation block contains a valid instance of the ``mango:EpochPosition`` class, the dynamic object describing the mapped data can generate a valid SkyCoord instance. [#591] +- RegTAP constraints involving tables other than rr.resource are now + done via subqueries for less duplication of interfaces. [#562, #572] + + - New sub-package discover for global dataset discovery. [#470] - Updated getdatalink to be consistent with iter_datalinks. [#613] - + + Deprecations and Removals ------------------------- diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index c86b6df8a..9bd156006 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -1,8 +1,8 @@ -******************** -MIVOT (`pyvo.mivot`) -******************** +********************** +MIVOT (``pyvo.mivot``) +********************** -This module contains the new feature of annotations in VOTable. +This module contains the new feature of handling model annotations in VOTable. Astropy version >= 6.0 is required. Introduction @@ -28,7 +28,7 @@ Introduction Implementation Scope --------------------- +==================== This implementation is totally model-agnostic. - It does not operate any validation against specific data models. @@ -42,8 +42,8 @@ Some of the examples have been provided by a special end-point of the Vizier con (https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch) that maps query results to this model. .. image:: _images/mangoEpochPosition.png - :width: 500 - :alt: EpochPropagation class used to validate this api. + :width: 500 + :alt: EpochPropagation class used to validate this api. It is to be noted that the Vizier service does not annotate errors at the time of writing (Q1 2024) @@ -56,191 +56,23 @@ which allows to get (and set) Mivot blocks from/into VOTables as an XML element epoch propagation use case: - ``JOIN`` features are not supported. - - ``TEMPLATES`` with more than one ``INSANCE`` not supported. - -Integrated Readout ------------------- -The ``ModelViewer`` module manages access to data mapped to a model through dynamically -generated objects (``MivotInstance``class). -The example below shows how a VOTable, resulting from a cone-search query which data are mapped -to the ``EpochPosition`` class, can be consumed. - -.. doctest-remote-data:: - >>> import astropy.units as u - >>> from astropy.coordinates import SkyCoord - >>> from pyvo.dal.scs import SCSService - >>> from pyvo.utils.prototype import activate_features - >>> from pyvo.mivot.version_checker import check_astropy_version - >>> from pyvo.mivot.viewer.mivot_viewer import MivotViewer - >>> activate_features("MIVOT") - >>> if check_astropy_version() is False: - ... pytest.skip("MIVOT test skipped because of the astropy version.") - >>> scs_srv = SCSService("https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch/I/239/hip_main") - >>> m_viewer = MivotViewer( - ... scs_srv.search( - ... pos=SkyCoord(ra=52.26708 * u.degree, dec=59.94027 * u.degree, frame='icrs'), - ... radius=0.05 - ... ) - ... ) - >>> mivot_instance = m_viewer.dm_instance - >>> print(mivot_instance.dmtype) - mango:EpochPosition - >>> print(mivot_instance.coordSys.spaceRefFrame.value) - ICRS - >>> while m_viewer.next(): - ... print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}") - position: 59.94033461 52.26722684 - - -In this example, the data readout is totally managed by the ``MivotViewer`` instance. -The ``astropy.io.votable`` API is encapsulated in this module. - -Model leaves (class attributes) are complex types that provide additional information: - -- ``value``: attribute value -- ``dmtype``: attribute type such as defined in the Mivot annotations -- ``unit``: attribute unit such as defined in the Mivot annotations -- ``ref``: identifier of the table column mapped on the attribute - -The model view on a data row can also be passed as a Python dictionary -using the ``to_dict()`` method of ``MivotInstance``. - -.. code-block:: python - :caption: Working with a model view as a dictionary - (the JSON layout has been squashed for display purpose) - - from pyvo.mivot import MivotViewer - from pyvo.mivot.utils.dict_utils import DictUtils - - m_viewer = MivotViewer(path_to_votable) - mivot_instance = m_viewer.dm_instance - mivot_object_dict = mivot_object.to_dict() - - DictUtils.print_pretty_json(mivot_object_dict) - { - "dmtype": "mango:EpochPosition", - "longitude": {"value": 359.94372764, "unit": "deg"}, - "latitude": {"value": -0.28005255, "unit": "deg"}, - "pmLongitude": {"value": -5.14, "unit": "mas/yr"}, - "pmLatitude": {"value": -25.43, "unit": "mas/yr"}, - "epoch": {"value": 1991.25, "unit": "year"}, - "coordSys": { - "dmtype": "coords:SpaceSys", - "dmid": "ICRS", - "dmrole": "coords:Coordinate.coordSys", - "spaceRefFrame": {"value": "ICRS"}, - }, - } - -- It is recommended to use a copy of the - dictionary as it will be rebuilt each time the ``to_dict()`` method is invoked. -- The default representation of ``MivotInstance`` instances is made with a pretty - string serialization of this dictionary (method ``__repr__()``). -- An extended version of the object dictionary e.g. with information about where - the values were picked from from, is available using the method ``to_hk_dict()``. - -Per-Row Readout ---------------- - -The annotation schema can also be applied to table rows read outside of the ``MivotViewer`` -with the `astropy.io.votable` API: - -.. code-block:: python - :caption: Accessing the model view of Astropy table rows - - votable = parse(path_to_votable) - table = votable.resources[0].tables[0] - # init the viewer - mivot_viewer = MivotViewer(votable, resource_number=0) - mivot_object = mivot_viewer.dm_instance - # and feed it with the table row - read = [] - for rec in table.array: - mivot_object.update(rec) - read.append(mivot_object.longitude.value) - # show that the model retrieve the correct data values - assert rec["RAICRS"] == mivot_object.longitude.value - assert rec["DEICRS"] == mivot_object.latitude.value - -In this case, it is up to the user to ensure that the read data rows are those mapped by the Mivot annotations. - -Get a SkyCoord Instance Directly From the Annotations ------------------------------------------------------ - -Once you get a ``MivotInstance`` representing the last row read, you can use it to create an ``astropy.SkyCoord`` object. - -.. code-block:: python - :caption: Accessing the model view of Astropy table rows - - from pyvo.mivot import MivotViewer - - m_viewer = MivotViewer(path_to_votable) - mivot_instance = m_viewer.dm_instance - print(mivot_instance.get_SkyCoord()) - - -This feature works under the condition that the annotations contain a valid instance of ``mango:EPochPosition``, otherwise -a ``NoMatchingDMTypeError`` is thrown. -Although not a standard at the time of writing, the class structure supported by this implementation must match the figure above. - - -For XML Hackers ---------------- - -The model instances can also be serialized as XML elements that can be parsed with XPath queries. - -.. code-block:: python - :caption: Accessing the XML view of the mapped model instances - - with MivotViewer(path_to_votable) as mivot_viewer: - while mivot_viewer.next(): - xml_view = mivot_viewer.xml_view - # do whatever you want with this XML element - -It to be noted that ``mivot_viewer.xml_view`` is a shortcut -for ``mivot_viewer.xml_view.view`` where ``mivot_viewer.xml_view`` -is is an instance of ``pyvo.mivot.viewer.XmlViewer``. -This object provides many functions facilitating the XML parsing. - -Class Generation in a Nutshell ------------------------------- - -MIVOT reconstructs model structures with 3 elements: - -- ``INSTANCE`` for the objects -- ``ATTRIBUTE`` for the attributes -- ``COLLECTION`` for the elements with a cardinality greater than 1 - -The role played by each of these elements in the model hierarchy is defined -by its ``@dmrole`` XML attribute. Types of both ``INSTANCE`` and ``ATTRIBUTE`` are defined by -their ``@dmtype`` XML attributes. - -``MivotInstance`` classes are built by following MIVOT annotation structure: - -- ``INSTANCE`` are represented by Python classes -- ``ATTRIBUTE`` are represented by Python class fields -- ``COLLECTION`` are represented by Python lists ([]) - -``@dmrole`` and ``@dmtype`` cannot be used as Python keywords as such, because they are built from VO-DML -identifiers, which have the following structure: ``model:a.b``. - -- Only the last part of the path is kept for attribute names. -- For class names, forbidden characters (``:`` or ``.``) are replaced with ``_``. -- Original ``@dmtype`` are kept as attributes of generated Python objects. -- The structure of the ``MivotInstance`` objects can be inferred from the mapped model in 2 different ways: + - ``TEMPLATES`` with more than one ``INSTANCE`` not supported. + + +Using the MIVOT package +======================= + +The ``pyvo.mivot`` module can be used to either read or build annotations. + +.. toctree:: + :maxdepth: 2 + + viewer + writer - - 1. From the MIVOT instance property ``MivotInstance.to_dict()`` a shown above. - This is a pure Python dictionary but its access can be slow because it is generated - on the fly each time the property is invoked. - - 2. From the internal class dictionary ``MivotInstance.__dict__`` - (see the Python `data model `_). Reference/API ============= .. automodapi:: pyvo.mivot -.. automodapi:: pyvo.mivot.viewer -.. automodapi:: pyvo.mivot.seekers -.. automodapi:: pyvo.mivot.features -.. automodapi:: pyvo.mivot.utils + diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst new file mode 100644 index 000000000..2df47ed9b --- /dev/null +++ b/docs/mivot/viewer.rst @@ -0,0 +1,199 @@ +***************************************** +MIVOT (``pyvo.mivot``): Annotation Viewer +***************************************** + + +Introduction +============ +.. pull-quote:: + + Model Instances in VOTables (MIVOT) defines a syntax to map VOTable + data to any model serialized in VO-DML. The annotation operates as a + bridge between the data and the model. It associates the column/param + metadata from the VOTable to the data model elements (class, attributes, + types, etc.) [...]. + The data model elements are grouped in an independent annotation block + complying with the MIVOT XML syntax. This annotation block is added + as an extra resource element at the top of the VOTable result resource. The + MIVOT syntax allows to describe a data structure as a hierarchy of classes. + It is also able to represent relations and composition between them. It can + also build up data model objects by aggregating instances from different + tables of the VOTable (get more in :doc:`index`). + +Using the API +============= + +Integrated Readout +------------------ +The ``ModelViewer`` module manages access to data mapped to a model through dynamically +generated objects (``MivotInstance`` class). + +The example below shows how a VOTable result of a cone-search query can be parsed and data +mapped to the ``EpochPosition`` class. + + +.. doctest-remote-data:: + >>> import astropy.units as u + >>> from astropy.coordinates import SkyCoord + >>> from pyvo.dal.scs import SCSService + >>> from pyvo.utils.prototype import activate_features + >>> from pyvo.mivot.version_checker import check_astropy_version + >>> from pyvo.mivot.viewer.mivot_viewer import MivotViewer + >>> activate_features("MIVOT") + >>> if check_astropy_version() is False: + ... pytest.skip("MIVOT test skipped because of the astropy version.") + >>> scs_srv = SCSService("https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch/I/239/hip_main") + >>> m_viewer = MivotViewer( + ... scs_srv.search( + ... pos=SkyCoord(ra=52.26708 * u.degree, dec=59.94027 * u.degree, frame='icrs'), + ... radius=0.05 + ... ) + ... ) + >>> mivot_instance = m_viewer.dm_instance + >>> print(mivot_instance.dmtype) + mango:EpochPosition + >>> print(mivot_instance.coordSys.spaceRefFrame.value) + ICRS + >>> while m_viewer.next(): + ... print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}") + position: 59.94033461 52.26722684 + + +In this example, the data readout is totally managed by the ``MivotViewer`` instance. +The ``astropy.io.votable`` API is encapsulated in this module. + +Model leaves (class attributes) are complex types that provide additional information: + +- ``value``: attribute value +- ``dmtype``: attribute type such as defined in the Mivot annotations +- ``unit``: attribute unit such as defined in the Mivot annotations +- ``ref``: identifier of the table column mapped on the attribute + +The model view on a data row can also be passed as a Python dictionary +using the ``dict`` property of ``MivotInstance``. + +.. code-block:: python + :caption: Working with a model view as a dictionary + (the JSON layout has been squashed for display purpose) + + from pyvo.mivot.utils.dict_utils import DictUtils + + mivot_instance = m_viewer.dm_instance + mivot_object_dict = mivot_object.dict + + DictUtils.print_pretty_json(mivot_object_dict) + { + "dmtype": "EpochPosition", + "longitude": {"value": 359.94372764, "unit": "deg"}, + "latitude": {"value": -0.28005255, "unit": "deg"}, + "pmLongitude": {"value": -5.14, "unit": "mas/yr"}, + "pmLatitude": {"value": -25.43, "unit": "mas/yr"}, + "epoch": {"value": 1991.25, "unit": "year"}, + "Coordinate_coordSys": { + "dmtype": "SpaceSys", + "dmid": "SpaceFrame_ICRS", + "dmrole": "coordSys", + "spaceRefFrame": {"value": "ICRS"}, + }, + } + +- It is recommended to use a copy of the + dictionary as it will be rebuilt each time the ``dict`` property is invoked. +- The default representation of ``MivotInstance`` instances is made with a pretty + string serialization of this dictionary. + +Per-Row Readout +--------------- + +The annotation schema can also be applied to table rows read outside of the ``MivotViewer`` +with the `astropy.io.votable` API: + +.. code-block:: python + :caption: Accessing the model view of Astropy table rows + + votable = parse(path_to_votable) + table = votable.resources[0].tables[0] + # init the viewer + mivot_viewer = MivotViewer(votable, resource_number=0) + mivot_object = mivot_viewer.dm_instance + # and feed it with the table row + read = [] + for rec in table.array: + mivot_object.update(rec) + read.append(mivot_object.longitude.value) + # show that the model retrieve the correct data values + assert rec["RAICRS"] == mivot_object.longitude.value + assert rec["DEICRS"] == mivot_object.latitude.value + +In this case, it is up to the user to ensure that the read data rows are those mapped by the Mivot annotations. + +For XML Hackers +--------------- + +The model instances can also be serialized as XML elements that can be parsed with XPath queries. + +.. code-block:: python + :caption: Accessing the XML view of the mapped model instances + + with MivotViewer(path_to_votable) as mivot_viewer: + while mivot_viewer.next(): + xml_view = mivot_viewer.xml_view + # do whatever you want with this XML element + +It is to be noted that ``mivot_viewer.xml_view`` is a shortcut +for ``mivot_viewer.xml_view.view`` where ``mivot_viewer.xml_view`` +is is an instance of ``pyvo.mivot.viewer.XmlViewer``. +This object provides many functions facilitating the XML parsing. + +Class Generation in a Nutshell +------------------------------ + +MIVOT reconstructs model structures with 3 elements: + +- ``INSTANCE`` for the objects +- ``ATTRIBUTE`` for the attributes +- ``COLLECTION`` for the elements with a cardinality greater than 1 + +The role played by each of these elements in the model hierarchy is defined +by its ``@dmrole`` XML attribute. Types of both ``INSTANCE`` and ``ATTRIBUTE`` are defined by +their ``@dmtype`` XML attributes. + +``MivotInstance`` classes are built by following MIVOT annotation structure: + +- ``INSTANCE`` are represented by Python classes +- ``ATTRIBUTE`` are represented by Python class fields +- ``COLLECTION`` are represented by Python lists ([]) + +``@dmrole`` and ``@dmtype`` cannot be used as Python keywords as such, because they are built from VO-DML +identifiers, which have the following structure: ``model:a.b``. + +- Only the last part of the path is kept for attribute names. +- For class names, forbidden characters (``:`` or ``.``) are replaced with ``_``. +- Original ``@dmtype`` are kept as attributes of generated Python objects. +- The structure of the ``MivotInstance`` objects can be inferred from the mapped model in 2 different ways: + + - 1. From the MIVOT instance property ``MivotInstance.dict`` a shown above. + This is a pure Python dictionary but its access can be slow because it is generated + on the fly each time the property is invoked. + - 2. From the internal class dictionary ``MivotInstance.__dict__`` + (see the Python `data model `_). + + .. code-block:: python + :caption: Exploring the MivotInstance structure with the internal dictionaries + + mivot_instance = mivot_viewer.dm_instance + + print(mivot_instance.__dict__.keys()) + dict_keys(['dmtype', 'longitude', 'latitude', 'pmLongitude', 'pmLatitude', 'epoch', 'Coordinate_coordSys']) + + print(mivot_instance.Coordinate_coordSys.__dict__.keys()) + dict_keys(['dmtype', 'dmid', 'dmrole', 'spaceRefFrame']) + + print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys()) + dict_keys(['dmtype', 'value', 'unit', 'ref']) + +Reference/API +============= + +.. automodapi:: pyvo.mivot.viewer + diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst new file mode 100644 index 000000000..ad8b8323d --- /dev/null +++ b/docs/mivot/writer.rst @@ -0,0 +1,235 @@ +***************************************** +MIVOT (``pyvo.mivot``): Annotation Writer +***************************************** + +Introduction +============ +.. pull-quote:: + + Model Instances in VOTables (MIVOT) defines a syntax to map VOTable + data to any model serialized in VO-DML. The annotation operates as a + bridge between the data and the model. It associates the column/param + metadata from the VOTable to the data model elements (class, attributes, + types, etc.) [...]. + The data model elements are grouped in an independent annotation block + complying with the MIVOT XML syntax. This annotation block is added + as an extra resource element at the top of the VOTable result resource. The + MIVOT syntax allows to describe a data structure as a hierarchy of classes. + It is also able to represent relations and composition between them. It can + also build up data model objects by aggregating instances from different + tables of the VOTable (get more in :doc:`index`). + +- Model Instances in VOTables is a VO `standard `_ +- Requires Astropy>=6.0 +- ``pyvo.mivot`` is a prototype feature which must be activated with ``activate_features("MIVOT")`` + +Using the API +============= + +Building Annotation Object per Object +------------------------------------- + +This documentation is intended for developers of data model classes who want to map them to VOTables +and not for end users. A future version will allow end users to create annotations with +ready-to-use data model building blocks. + +Creating annotations consists of 3 steps: + +#. Create individual instances (INSTANCE) using the ``MivotInstance`` class: objects are + built attribute by attribute. These components can then be aggregated into + more complex objects following the structure of the mapped model(s). +#. Wrap the annotations with the ``MivotAnnotations`` class: declare to the annotation builder + the models used, and place individual instances at the right place (TEMPLATES or GLOBALS). +#. Insert the annotations into a VOtable by using the Astropy API (wrapped in the package logic). + +The annotation builder does not check whether the XML conforms to any particular model. +It simply validates it against the MIVOT XML Schema if the ``xmlvalidator`` package if is installed. + +The example below shows a step-by-step construction of a MIVOT block mapping +a position with its error (as defined in the ``MANGO`` draft) +and its space coordinate system (as defined in the ``coord`` model and imported by ``MANGO``). + + +Building the Coordinate System Object +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The space coordinate system is made of a space frame and a reference position, both wrapped in a ``coords:SpaceSys`` +object (see the `coords `_ model). + +- Individual instances are built one by one and then packed together. +- The top level object has a ``dmid`` which will be used as a reference by the ``EpochPosition`` instance. +- ``MIVOT`` is still an experimental feature which must be activated + +.. code-block:: python + + + from astropy.io.votable import parse + from pyvo.utils import activate_features + from pyvo.mivot.utils.exceptions import MappingException + from pyvo.mivot.utils.dict_utils import DictUtils + from pyvo.mivot.writer.annotations import MivotAnnotations + from pyvo.mivot.writer.instance import MivotInstance + from pyvo.mivot.viewer.mivot_viewer import MivotViewer + + activate_features("MIVOT") + + space_sys = MivotInstance(dmid="_spacesys_icrs", dmtype="c") + space_frame = MivotInstance( + dmrole="coords:PhysicalCoordSys.frame", dmtype="coords:SpaceFrame" + ) + space_frame.add_attribute( + dmrole="coords:SpaceFrame.spaceRefFrame", dmtype="ivoa:string", value="ICRS" + ) + ref_position = MivotInstance( + dmrole="coords:SpaceFrame.refPosition", dmtype="coords:StdRefLocation" + ) + ref_position.add_attribute( + dmrole="coords:StdRefLocation.position", + dmtype="ivoa:string", + value="BARYCENTER", + ) + space_frame.add_instance(ref_position) + space_sys.add_instance(space_frame) + + +Building the EpochPosition Object +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- In this example we only use the position attributes (RA/DEC) of the ``EpochPosition`` class. +- The reference to the space coordinate system is added at the end. +- The ``ref`` XML attributes reference columns that must be used to set the model attributes. + Their values depend on the VOTable to be mapped. + +.. code-block:: python + + + position = MivotInstance(dmtype="mango:EpochPosition") + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.longitude", + unit="deg", + ref="RAICRS", + ) + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.latitude", + unit="deg", + ref="DEICRS", + ) + position.add_reference( + dmref="_spacesys_icrs", dmrole="mango:EpochPosition.spaceSys" + ) + + +Building the Position Error +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- We assume that the position error is the same on both axes without correlation. + In terms of MANGO error, this corresponds to a 2x2 diagonal error matrix with two equal coefficients. +- Finally, the error is added as a component of the ``EpochPosition`` instance. + +.. code-block:: python + + + epoch_position_error = MivotInstance( + dmtype="mango:EpochPositionErrors", dmrole="mango:EpochPosition.errors" + ) + position_error = MivotInstance( + dmtype="mango:error.ErrorCorrMatrix", + dmrole="mango:EpochPositionErrors.position", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma1", + unit="arcsec", + ref="sigm", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma2", + unit="arcsec", + ref="sigm", + ) + epoch_position_error.add_instance(position_error) + position.add_instance(epoch_position_error) + + +Building the MIVOT Block +^^^^^^^^^^^^^^^^^^^^^^^^ + +- The MIVOT block consists of: + + - A process status + - A list of mapped models + - A list of globals, which are objects not associated with + VOTable data and that can be shared by any other MIVOT instance. + - A list of templates, which are objects that are connected to + VOTable data and whose leaf values change from one row to another. + +- The latest step (build_mivot_block) includes a validation of the MIVOT syntax that works only + if the ``xmlvaldator`` package has been installed. + +.. code-block:: python + + + mivot_annotations = MivotAnnotations() + mivot_annotations.add_model( + "ivoa", "https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" + ) + mivot_annotations.add_model( + "coords", "https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" + ) + mivot_annotations.add_model( + "mango", + "https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", + ) + mivot_annotations.set_report(True, "PyVO Tuto") + + mivot_annotations.add_templates(position) + mivot_annotations.add_globals(space_sys) + + mivot_annotations.build_mivot_block() + + +Insert the MIVOT Block in a VOTable +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- This straightforward step is based on the Astropy VOTable API. +- Annotations are stored in-memory (in the parsed VOtable) +- The mapping can be tested with the ``MivotViewer`` API (see the :doc:`viewer`) +- The VOtable must be explicitly saved on disk if needed. + + .. code-block:: python + + + from astropy.io.votable import parse + + votable = parse(votable_path) + mivot_annotations.insert_into_votable(votable) + + mv = MivotViewer(votable) + mappes_instance = mv.dm_instance + + votable.to_xml("pyvo-tuto.xml") + + +Validate the annotation against the models +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- This action requires the ``mivot-validator`` package to be installed. +- It validates the mapped classes against the models they come from. + + + .. code-block:: shell + :caption: Build the coordinate system (coords:SpaceSys) + + % pip install mivot-validator + % mivot-instance-validate pyvo-tuto.xml + ... + Valid if no error message + ... + +Reference/API +============= + +.. automodapi:: pyvo.mivot.writer diff --git a/pyvo/mivot/__init__.py b/pyvo/mivot/__init__.py index c096ba11e..9dce85d06 100644 --- a/pyvo/mivot/__init__.py +++ b/pyvo/mivot/__init__.py @@ -1,2 +1 @@ -# package entry point -from .viewer.mivot_viewer import MivotViewer \ No newline at end of file +# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/pyvo/mivot/tests/data/essai.xml b/pyvo/mivot/tests/data/essai.xml new file mode 100644 index 000000000..8a752082b --- /dev/null +++ b/pyvo/mivot/tests/data/essai.xml @@ -0,0 +1,348 @@ + + + + + + URAT1 Catalog (Zacharias+ 2015) + + + + IVOID of underlying data collection + + + Mivot writer unit test + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + URAT1 catalog + + + + Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of + catalog (Epoch) + + + + + URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] + + + + + Right ascension on ICRS, at "Epoch" (1) + + + + + Declination on ICRS, at "Epoch" (1) + + + + + Position error per coordinate, from scatter (2) + + + + + Position error per coordinate, from model (2) + + + + + (nst) Total number of sets the star is in (3) + + + + + (nsu) Number of sets used for mean position (3) + + + + + (epoc) Mean URAT observation epoch (1) + + + + + ?(mmag) mean URAT model fit magnitude (4) + + + + + + ?(sigp) URAT photometry error (5) + + + + + + (nsm) Number of sets used for URAT magnitude (3) + + + + + (ref) largest reference star flag (6) + + + + + (nit) Total number of images (observations) + + + + + (niu) Number of images used for mean position + + + + + (ngt) Total number of 1st order grating observations + + + + + (ngu) Number of 1st order grating positions used + + + + + ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) + + + + + + ?(pmd) Proper motion in Declination (7) + + + + + + ?(pme) Proper motion error per coordinate (8) + + + + + + [1/11] Match flag URAT with 2MASS (9) + + + + + [1/11] Match flag URAT with APASS (9) + + + + + [-] "-" if there is no match with GSC2.4 (14) + + + + + ?(id2) unique 2MASS star identification number + + + + + + ?(jmag) 2MASS J-band magnitude + + + + + + ?(ejmag) Error on Jmag + + + + + + [0,58]? J-band quality-confusion flag (10) + + + + + ?(hmag) 2MASS H-band magnitude + + + + + + ?(ehmag) Error on H-band magnitude (10) + + + + + + [0,58]? H-band quality-confusion flag (10) + + + + + ?(kmag) 2MASS Ks-band magnitude + + + + + + ?(ekmag) Error on Ks-band magnitude (10) + + + + + + [0,58]? Ks-band quality-confusion flag (10) + + + + + (ann) Number of APASS observation nights (12) + + + + + (ano) Number of APASS observations (12) + + + + + ?(abm) APASS B-band magnitude (11) + + + + + + ?(ebm) Error on Bmag + + + + + + ?(avm) APASS V-band magnitude + + + + + + ?(evm) Error on Vmag + + + + + + ?(agm) APASS g-band magnitude + + + + + + ?(egm) Error on gmag + + + + + + ?(arm) APASS r-band magnitude + + + + + + ?(erm) Error on rmag + + + + + + ?(aim) APASS i-band magnitude + + + + + + ?(eim) Error on imag + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
0.049402750-146023 52.2340018 59.89373339613132013.41815.340 0.013139747400 1.5 -12.3 5.915 + 75880868113.713 0.028513.340 0.034513.101 0.03451417.632 0.20416.164 0.00116.690 0.00115.750 0.001 + +
+ + diff --git a/pyvo/mivot/tests/data/reference/test_mivot_writer.json b/pyvo/mivot/tests/data/reference/test_mivot_writer.json new file mode 100644 index 000000000..4e003b90e --- /dev/null +++ b/pyvo/mivot/tests/data/reference/test_mivot_writer.json @@ -0,0 +1,46 @@ +{ + "dmtype": "mango:EpochPosition", + "longitude": { + "value": 52.2340018, + "unit": "deg" + }, + "latitude": { + "value": 59.8937333, + "unit": "deg" + }, + "errors": { + "dmtype": "mango:EpochPositionErrors", + "dmrole": "mango:EpochPosition.errors", + "position": { + "dmtype": "mango:error.ErrorCorrMatrix", + "dmrole": "mango:EpochPositionErrors.position", + "sigma1": { + "value": 6.0, + "unit": "arcsec" + }, + "sigma2": { + "value": 6.0, + "unit": "arcsec" + } + } + }, + "spaceSys": { + "dmtype": "coords:SpaceSys", + "dmid": "_spacesys_icrs", + "dmrole": "mango:EpochPosition.spaceSys", + "frame": { + "dmtype": "coords:SpaceFrame", + "dmrole": "coords:PhysicalCoordSys.frame", + "spaceRefFrame": { + "value": "ICRS" + }, + "refPosition": { + "dmtype": "coords:StdRefLocation", + "dmrole": "coords:SpaceFrame.refPosition", + "position": { + "value": "BARYCENTER" + } + } + } + } +} \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/test_mivot_writer.xml b/pyvo/mivot/tests/data/reference/test_mivot_writer.xml new file mode 100644 index 000000000..62af6ef7d --- /dev/null +++ b/pyvo/mivot/tests/data/reference/test_mivot_writer.xml @@ -0,0 +1,29 @@ + + Mivot writer unit test + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml b/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml index 54c5b32f5..71a60fe0c 100644 --- a/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml +++ b/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml @@ -1,222 +1,317 @@ - - - - - URAT1 Catalog (Zacharias+ 2015) IVOID of underlying data collection - + + + + + + URAT1 Catalog (Zacharias+ 2015) + - - - URAT1 catalog - - - - Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of catalog (Epoch) - - - - URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] - - - - Right ascension on ICRS, at "Epoch" (1) - - - - Declination on ICRS, at "Epoch" (1) - - - - Position error per coordinate, from scatter (2) - - - - Position error per coordinate, from model (2) - - - - (nst) Total number of sets the star is in (3) - - - - (nsu) Number of sets used for mean position (3) - - - - (epoc) Mean URAT observation epoch (1) - - - - ?(mmag) mean URAT model fit magnitude (4) - - - - - ?(sigp) URAT photometry error (5) - - - - - (nsm) Number of sets used for URAT magnitude (3) - - - - (ref) largest reference star flag (6) - - - - (nit) Total number of images (observations) - - - - (niu) Number of images used for mean position - - - - (ngt) Total number of 1st order grating observations - - - - (ngu) Number of 1st order grating positions used - - - - ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) - - - - - ?(pmd) Proper motion in Declination (7) - - - - - ?(pme) Proper motion error per coordinate (8) - - - - - [1/11] Match flag URAT with 2MASS (9) - - - - [1/11] Match flag URAT with APASS (9) - - - - [-] "-" if there is no match with GSC2.4 (14) - - - - ?(id2) unique 2MASS star identification number - - - - - ?(jmag) 2MASS J-band magnitude - - - - - ?(ejmag) Error on Jmag - - - - - [0,58]? J-band quality-confusion flag (10) - - - - ?(hmag) 2MASS H-band magnitude - - - - - ?(ehmag) Error on H-band magnitude (10) - - - - - [0,58]? H-band quality-confusion flag (10) - - - - ?(kmag) 2MASS Ks-band magnitude - - - - - ?(ekmag) Error on Ks-band magnitude (10) - - - - - [0,58]? Ks-band quality-confusion flag (10) - - - - (ann) Number of APASS observation nights (12) - - - - (ano) Number of APASS observations (12) - - - - ?(abm) APASS B-band magnitude (11) - - - - - ?(ebm) Error on Bmag - - - - - ?(avm) APASS V-band magnitude - - - - - ?(evm) Error on Vmag - - - - - ?(agm) APASS g-band magnitude - - - - - ?(egm) Error on gmag - - - - - ?(arm) APASS r-band magnitude - - - - - ?(erm) Error on rmag - - - - - ?(aim) APASS i-band magnitude - - - - - ?(eim) Error on imag - - - - - - -
0.049402750-146023052.2340018+59.89373339613132013.41815.3400.0131397474001.5-12.35.91575880868113.7130.028513.3400.034513.1010.03451417.6320.20416.1640.00116.6900.00115.7500.001
- + + IVOID of underlying data collection + + + URAT1 catalog + + + + Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of + catalog (Epoch) + + + + + URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] + + + + + Right ascension on ICRS, at "Epoch" (1) + + + + + Declination on ICRS, at "Epoch" (1) + + + + + Position error per coordinate, from scatter (2) + + + + + Position error per coordinate, from model (2) + + + + + (nst) Total number of sets the star is in (3) + + + + + (nsu) Number of sets used for mean position (3) + + + + + (epoc) Mean URAT observation epoch (1) + + + + + ?(mmag) mean URAT model fit magnitude (4) + + + + + + ?(sigp) URAT photometry error (5) + + + + + + (nsm) Number of sets used for URAT magnitude (3) + + + + + (ref) largest reference star flag (6) + + + + + (nit) Total number of images (observations) + + + + + (niu) Number of images used for mean position + + + + + (ngt) Total number of 1st order grating observations + + + + + (ngu) Number of 1st order grating positions used + + + + + ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) + + + + + + ?(pmd) Proper motion in Declination (7) + + + + + + ?(pme) Proper motion error per coordinate (8) + + + + + + [1/11] Match flag URAT with 2MASS (9) + + + + + [1/11] Match flag URAT with APASS (9) + + + + + [-] "-" if there is no match with GSC2.4 (14) + + + + + ?(id2) unique 2MASS star identification number + + + + + + ?(jmag) 2MASS J-band magnitude + + + + + + ?(ejmag) Error on Jmag + + + + + + [0,58]? J-band quality-confusion flag (10) + + + + + ?(hmag) 2MASS H-band magnitude + + + + + + ?(ehmag) Error on H-band magnitude (10) + + + + + + [0,58]? H-band quality-confusion flag (10) + + + + + ?(kmag) 2MASS Ks-band magnitude + + + + + + ?(ekmag) Error on Ks-band magnitude (10) + + + + + + [0,58]? Ks-band quality-confusion flag (10) + + + + + (ann) Number of APASS observation nights (12) + + + + + (ano) Number of APASS observations (12) + + + + + ?(abm) APASS B-band magnitude (11) + + + + + + ?(ebm) Error on Bmag + + + + + + ?(avm) APASS V-band magnitude + + + + + + ?(evm) Error on Vmag + + + + + + ?(agm) APASS g-band magnitude + + + + + + ?(egm) Error on gmag + + + + + + ?(arm) APASS r-band magnitude + + + + + + ?(erm) Error on rmag + + + + + + ?(aim) APASS i-band magnitude + + + + + + ?(eim) Error on imag + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
0.049402750-146023 52.2340018 59.89373339613132013.41815.340 0.013139747400 1.5 -12.3 5.915 + 75880868113.713 0.028513.340 0.034513.101 0.03451417.632 0.20416.164 0.00116.690 0.00115.750 0.001 + +
+ diff --git a/pyvo/mivot/tests/test_annotation_seeker.py b/pyvo/mivot/tests/test_annotation_seeker.py index 86ad4c73f..fd741eed4 100644 --- a/pyvo/mivot/tests/test_annotation_seeker.py +++ b/pyvo/mivot/tests/test_annotation_seeker.py @@ -11,7 +11,7 @@ from pyvo.mivot.seekers.annotation_seeker import AnnotationSeeker from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from . import XMLOutputChecker diff --git a/pyvo/mivot/tests/test_mivot_instance.py b/pyvo/mivot/tests/test_mivot_instance.py index 1e314bf44..81607ff83 100644 --- a/pyvo/mivot/tests/test_mivot_instance.py +++ b/pyvo/mivot/tests/test_mivot_instance.py @@ -11,7 +11,7 @@ from pyvo.mivot.version_checker import check_astropy_version from pyvo.mivot.viewer.mivot_instance import MivotInstance from pyvo.mivot.utils.mivot_utils import MivotUtils -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer fake_hk_dict = { "dmtype": "EpochPosition", diff --git a/pyvo/mivot/tests/test_mivot_instance_generation.py b/pyvo/mivot/tests/test_mivot_instance_generation.py index cab728aed..ea2ffb85c 100644 --- a/pyvo/mivot/tests/test_mivot_instance_generation.py +++ b/pyvo/mivot/tests/test_mivot_instance_generation.py @@ -6,7 +6,7 @@ import pytest from urllib.request import urlretrieve from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from pyvo.mivot.utils.mivot_utils import MivotUtils diff --git a/pyvo/mivot/tests/test_mivot_viewer.py b/pyvo/mivot/tests/test_mivot_viewer.py index 20fb63f85..ff79da42b 100644 --- a/pyvo/mivot/tests/test_mivot_viewer.py +++ b/pyvo/mivot/tests/test_mivot_viewer.py @@ -10,7 +10,7 @@ from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.utils.exceptions import MappingError from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from astropy import version as astropy_version diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py new file mode 100644 index 000000000..21dd5c533 --- /dev/null +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -0,0 +1,198 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This module contains test cases for validating the functionality of MivotInstance, MivotAnnotations, +and related components in the pyvo.mivot package. These tests ensure that the classes behave as +expected, including error handling and XML generation for data models. +""" + +import os +import pytest +from astropy.io.votable import parse +from pyvo.utils import activate_features +from pyvo.mivot.version_checker import check_astropy_version +from pyvo.mivot.utils.exceptions import MappingError +from pyvo.mivot.utils.dict_utils import DictUtils +from pyvo.mivot.writer.annotations import MivotAnnotations +from pyvo.mivot.writer.instance import MivotInstance +from pyvo.mivot.viewer.mivot_viewer import MivotViewer + +# Enable MIVOT-specific features in the pyvo library +activate_features("MIVOT") + +# File paths for test data +votable_path = os.path.realpath( + os.path.join(__file__, "..", "data", "test.mivot_viewer.no_mivot.xml") +) +data_path = os.path.realpath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") +) + + +def strip_xml(xml_string): + """ + Strip unnecessary whitespace and newline characters from an XML string. + + Parameters: + - xml_string (str): The XML string to strip. + + Returns: + - str: The stripped XML string. + """ + return ( + xml_string.replace("\n", "").replace(" ", "").replace("'", "").replace('"', "") + ) + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_MivotInstance(): + """ + Test the MivotInstance class for various operations including attribute addition, + reference addition, and XML generation. Verifies that invalid operations raise + the expected MappingError. + """ + with pytest.raises(MappingError): + MivotInstance(dmid="model:type.inst") + + instance1 = MivotInstance(dmtype="model:type.inst", dmid="id1") + with pytest.raises(MappingError): + instance1.add_attribute( + dmrole="model:type.inst.role1", value="value1", unit="m/s" + ) + with pytest.raises(MappingError): + instance1.add_attribute( + dmtype="model:type.att1", dmrole="model:type.inst.role1" + ) + with pytest.raises(MappingError): + instance1.add_reference(dmref="dmreference") + with pytest.raises(MappingError): + instance1.add_reference(dmrole="model:type.inst.role2") + with pytest.raises(MappingError): + instance1.add_instance("azerty") + + instance1.add_reference(dmrole="model:type.inst.role2", dmref="dmreference") + instance1.add_attribute( + dmtype="model:type.att1", + dmrole="model:type.inst.role1", + value="value1", + unit="m/s", + ) + assert strip_xml(instance1.xml_string()) == ( + "" + + "" + + "" + + "" + ) + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_MivotAnnotations(): + """ + Test the MivotAnnotations class for template and global instance addition. Verifies + that invalid operations raise the expected MappingError. + """ + mb = MivotAnnotations() + + with pytest.raises(MappingError): + mb.add_templates(12) + with pytest.raises(MappingError): + mb.add_globals(12) + + +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_MivotInstanceAll(): + """ + Test the creation and combination of multiple MivotInstance objects, their attributes, + references, and integration into a MivotAnnotations instance. Verifies correct XML + generation and VOTable integration. + """ + + space_sys = MivotInstance(dmid="_spacesys_icrs", dmtype="coords:SpaceSys") + space_frame = MivotInstance( + dmrole="coords:PhysicalCoordSys.frame", dmtype="coords:SpaceFrame" + ) + space_frame.add_attribute( + dmrole="coords:SpaceFrame.spaceRefFrame", dmtype="ivoa:string", value="ICRS" + ) + ref_position = MivotInstance( + dmrole="coords:SpaceFrame.refPosition", dmtype="coords:StdRefLocation" + ) + ref_position.add_attribute( + dmrole="coords:StdRefLocation.position", + dmtype="ivoa:string", + value="BARYCENTER", + ) + space_frame.add_instance(ref_position) + space_sys.add_instance(space_frame) + + position = MivotInstance(dmtype="mango:EpochPosition") + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.longitude", + unit="deg", + ref="RAICRS", + ) + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.latitude", + unit="deg", + ref="DEICRS", + ) + + epoch_position_error = MivotInstance( + dmtype="mango:EpochPositionErrors", dmrole="mango:EpochPosition.errors" + ) + position_error = MivotInstance( + dmtype="mango:error.ErrorCorrMatrix", + dmrole="mango:EpochPositionErrors.position", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma1", + unit="arcsec", + ref="sigm", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma2", + unit="arcsec", + ref="sigm", + ) + epoch_position_error.add_instance(position_error) + position.add_reference( + dmref="_spacesys_icrs", dmrole="mango:EpochPosition.spaceSys" + ) + position.add_instance(epoch_position_error) + + mivot_annotations = MivotAnnotations() + mivot_annotations.add_model( + "ivoa", + vodml_url="https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" + ) + mivot_annotations.add_model( + "coords", + vodml_url="https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" + ) + mivot_annotations.add_model( + "mango", + vodml_url="https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", + ) + mivot_annotations.set_report(True, "Mivot writer unit test") + + mivot_annotations.add_templates(position) + mivot_annotations.add_globals(space_sys) + + mivot_annotations.build_mivot_block() + with open( + os.path.join(data_path, "reference/test_mivot_writer.xml"), "r" + ) as xml_ref: + assert strip_xml(xml_ref.read()) == strip_xml(mivot_annotations.mivot_block) + + votable = parse(votable_path) + mivot_annotations.insert_into_votable(votable) + + mv = MivotViewer(votable) + print(mv.dm_instance) + assert mv.dm_instance.to_dict() == DictUtils.read_dict_from_file( + os.path.join(data_path, "reference/test_mivot_writer.json") + ) + votable.to_xml(data_path + "/essai.xml") diff --git a/pyvo/mivot/tests/test_static_reference.py b/pyvo/mivot/tests/test_static_reference.py index 82bce325e..1c5096f53 100644 --- a/pyvo/mivot/tests/test_static_reference.py +++ b/pyvo/mivot/tests/test_static_reference.py @@ -7,7 +7,7 @@ from pyvo.mivot.seekers.annotation_seeker import AnnotationSeeker from pyvo.mivot.features.static_reference_resolver import StaticReferenceResolver from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from . import XMLOutputChecker diff --git a/pyvo/mivot/tests/test_user_api.py b/pyvo/mivot/tests/test_user_api.py index 04277c0e6..67a74a804 100644 --- a/pyvo/mivot/tests/test_user_api.py +++ b/pyvo/mivot/tests/test_user_api.py @@ -12,7 +12,7 @@ from astropy.coordinates import SkyCoord from pyvo.dal.scs import SCSService from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from astropy.io.votable import parse from pyvo.mivot.utils.dict_utils import DictUtils diff --git a/pyvo/mivot/tests/test_vizier_cs.py b/pyvo/mivot/tests/test_vizier_cs.py index 573430cec..72fff1cf0 100644 --- a/pyvo/mivot/tests/test_vizier_cs.py +++ b/pyvo/mivot/tests/test_vizier_cs.py @@ -20,7 +20,7 @@ import pytest from urllib.request import urlretrieve from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from pyvo.mivot.utils.exceptions import MivotError diff --git a/pyvo/mivot/tests/test_xml_viewer.py b/pyvo/mivot/tests/test_xml_viewer.py index e0dde04ea..8ff8f6671 100644 --- a/pyvo/mivot/tests/test_xml_viewer.py +++ b/pyvo/mivot/tests/test_xml_viewer.py @@ -9,7 +9,7 @@ from xml.etree.ElementTree import Element as element from astropy.utils.data import get_pkg_data_filename from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from pyvo.mivot.utils.exceptions import MivotError diff --git a/pyvo/mivot/utils/xml_utils.py b/pyvo/mivot/utils/xml_utils.py index 83f15a108..a35676986 100644 --- a/pyvo/mivot/utils/xml_utils.py +++ b/pyvo/mivot/utils/xml_utils.py @@ -23,12 +23,13 @@ def pretty_print(xmltree): print(XmlUtils.pretty_string(xmltree)) @staticmethod - def pretty_string(xmltree): + def pretty_string(xmltree, clean_namespace=True): """ Return a pretty string representation of an XML tree. Parameters ---------- - xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string. + xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string + clean_namespace (boolean): Ddefault namspace (ns0) removed from element names if True Returns ------- str: The pretty string representation of the XML tree. @@ -39,7 +40,10 @@ def pretty_string(xmltree): else: XmlUtils.indent(xmltree) new_xml = ET.tostring(xmltree, encoding='unicode') - return new_xml.replace("ns0:", "") + if clean_namespace: + return new_xml.replace("ns0:", "") + else: + return new_xml @staticmethod def indent(elem, level=0): diff --git a/pyvo/mivot/viewer/__init__.py b/pyvo/mivot/viewer/__init__.py index e69de29bb..0ca6afaf7 100644 --- a/pyvo/mivot/viewer/__init__.py +++ b/pyvo/mivot/viewer/__init__.py @@ -0,0 +1,3 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +from .mivot_instance import MivotInstance +from .mivot_viewer import MivotViewer diff --git a/pyvo/mivot/writer/__init__.py b/pyvo/mivot/writer/__init__.py new file mode 100644 index 000000000..f0f23bf23 --- /dev/null +++ b/pyvo/mivot/writer/__init__.py @@ -0,0 +1,3 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +from .annotations import MivotAnnotations +from .instance import MivotInstance \ No newline at end of file diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py new file mode 100644 index 000000000..eac6b446f --- /dev/null +++ b/pyvo/mivot/writer/annotations.py @@ -0,0 +1,342 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +MivotAnnotations: A utility module to build and manage MIVOT annotations. +""" +import os +import logging + +try: + import xmlschema +except ImportError: + xmlschema = None +# Use defusedxml only if already present in order to avoid a new depency. +try: + from defusedxml import ElementTree as etree +except ImportError: + from xml.etree import ElementTree as etree +from astropy.io.votable.tree import VOTableFile, Resource +try: + from astropy.io.votable.tree import MivotBlock +except ImportError: + pass +from astropy.io.votable import parse +from astropy import version +from pyvo.utils.prototype import prototype_feature +from pyvo.mivot.utils.xml_utils import XmlUtils +from pyvo.mivot.utils.exceptions import MappingError, AstropyVersionException +from pyvo.mivot.writer.instance import MivotInstance +from pyvo.mivot.version_checker import check_astropy_version + +__all__ = ["MivotAnnotations"] + + +@prototype_feature("MIVOT") +class MivotAnnotations: + """ + This module provides a class to construct, validate, and insert MIVOT + blocks into VOTable files. + The MIVOT block, represented as an XML structure, is used for + data model annotations in the IVOA ecosystem. + + The main features are: + + - Construct the MIVOT block step-by-step with various components. + - Validate the MIVOT block against the MIVOT XML schema (if ``xmlschema`` is installed). + - Embed the MIVOT block into an existing VOTable file. + + The MIVOT block is constructed as a string to maintain compatibility with the Astropy API. + + Attributes + ---------- + _models : dict + A dictionary containing models with their names as keys and URLs as values. + _report_status : bool + Indicates the success status of the annotation process. + _report_message : str + A message associated with the report, used in the REPORT block. + _globals : list + A list of GLOBALS blocks to be included in the MIVOT block. + _templates : list + A list of TEMPLATES blocks to be included in the MIVOT block. + _templates_id : str + An optional ID for the TEMPLATES block. + _mivot_block : str + The complete MIVOT block as a string. + """ + + def __init__(self): + """ + """ + self._models = {} + self._report_status = True + self._report_message = "Generated by pyvo.mivot.writer" + self._globals = [] + self._templates = [] + self._templates_id = "" + self._mivot_block = "" + + @property + def mivot_block(self): + """ + Getter for the whole MIVOT block. + + Returns + ------- + str + Complete MIVOT block as a string. + """ + return self._mivot_block + + def _get_report(self): + """ + Generate the component of the MIVOT block. + + Returns + ------- + str + The block as a string, indicating the success or failure of the process. + """ + if self._report_status: + return f'{self._report_message}' + else: + return f'{self._report_message}' + + def _get_models(self): + """ + Generate the components of the MIVOT block. + + Returns + ------- + str + The components as a formatted string. + """ + models_block = "" + for key, value in self._models.items(): + if value: + models_block += f'\n' + else: + models_block += f'\n' + + return models_block + + def _get_globals(self): + """ + Generate the component of the MIVOT block. + + Returns + ------- + str + The block as a formatted string. + """ + globals_block = "\n" + for glob in self._globals: + globals_block += f"{glob}\n" + globals_block += "\n" + + return globals_block + + def _get_templates(self): + """ + Generate the component of the MIVOT block. + + Returns + ------- + str + The block as a formatted string, or an empty string if no templates are defined. + """ + if not self._templates: + return "" + if not self._templates_id: + templates_block = "\n" + else: + templates_block = f'\n' + + for templates in self._templates: + templates_block += f"{templates}\n" + templates_block += "\n" + return templates_block + + def build_mivot_block(self, *, templates_id=None): + """ + Build a complete MIVOT block from the declared components and validates it + against the MIVOT XML schema. + + Parameters + ---------- + templates_id : str, optional + The ID to associate with the block. Defaults to None. + + Raises + ------ + Any exceptions raised during XML validation are not caught and must + be handled by the caller. + """ + if templates_id: + self._templates_id = templates_id + self._mivot_block = '\n' + self._mivot_block += self._get_report() + self._mivot_block += "\n" + self._mivot_block += self._get_models() + self._mivot_block += "\n" + self._mivot_block += self._get_globals() + self._mivot_block += "\n" + self._mivot_block += self._get_templates() + self._mivot_block += "\n" + self._mivot_block += "\n" + self._mivot_block = self.mivot_block.replace("\n\n", "\n") + self.check_xml() + + def add_templates(self, templates_instance): + """ + Add an element to the block. + + Parameters + ---------- + templates_instance : str or MivotInstance + The element to be added. + + Raises + ------ + MappingError + If ``templates_instance`` is neither a string nor an instance of `MivotInstance`. + """ + if isinstance(templates_instance, MivotInstance): + self._templates.append(templates_instance.xml_string()) + elif isinstance(templates_instance, str): + self._templates.append(templates_instance) + else: + raise MappingError( + "Instance added to templates must be a string or MivotInstance." + ) + + def add_globals(self, globals_instance): + """ + Add an block to the block. + + Parameters + ---------- + globals_instance : str or MivotInstance + The block to be added. + + Raises + ------ + MappingError + If ``globals_instance`` is neither a string nor an instance of `MivotInstance`. + """ + if isinstance(globals_instance, MivotInstance): + self._globals.append(globals_instance.xml_string()) + elif isinstance(globals_instance, str): + self._globals.append(globals_instance) + else: + raise MappingError( + "Instance added to globals must be a string or MivotInstance." + ) + + def add_model(self, model_name, *, vodml_url=None): + """ + Add a element to the MIVOT block. + + Parameters + ---------- + model_name : str + The short name of the model. + vodml_url : str, optional + The URL of the VO-DML file associated with the model. + """ + self._models[model_name] = vodml_url + + def set_report(self, status, message): + """ + Set the element of the MIVOT block. + + Parameters + ---------- + status : bool + The status of the annotation process. True for success, False for failure. + message : str + The message associated with the REPORT. + + Notes + ----- + If ``status`` is False, all components of the MIVOT block except MODEL and REPORT + are cleared. + """ + self._report_status = status + self._report_message = message + if not status: + self._globals = [] + self._templates = [] + + def check_xml(self): + """ + Validate the MIVOT block against the MIVOT XML schema v1.0. + + Raises + ------ + MappingError + If the validation fails. + + Notes + ----- + The schema (mivot 1.0) is loaded from a local file to avoid dependency on a remote service. + """ + # put here just to improve the test coverage + root = etree.fromstring(self._mivot_block) + mivot_block = XmlUtils.pretty_string(root, clean_namespace=False) + if not xmlschema: + logging.error( + "XML validation skipped: no XML schema found. " + + "Please install it (e.g., pip install xmlschema)." + ) + return + + schema = xmlschema.XMLSchema11(os.path.dirname(__file__) + "/mivot-v1.xsd") + + try: + schema.validate(mivot_block) + except Exception as excep: + raise MappingError(f"Validation failed: {excep}") from excep + + def insert_into_votable(self, votable_file, override=False): + """ + Insert the MIVOT block into a VOTable. + + Parameters + ---------- + votable_file : str or VOTableFile + The VOTable to be annotated, either as a file path or a ``VOTableFile`` instance. + override : bool + If True, overrides any existing annotations in the VOTable. + + Raises + ------ + MappingError + If a mapping block already exists and ``override`` is False. + """ + if not check_astropy_version(): + raise AstropyVersionException(f"Astropy version {version.version} " + f"is below the required version 6.0 for the use of MIVOT.") + if isinstance(votable_file, str): + votable = parse(votable_file) + elif isinstance(votable_file, VOTableFile): + votable = votable_file + else: + raise MappingError( + "votable_file must be a file path string or a VOTableFile instance." + ) + + for resource in votable.resources: + if resource.type == "results": + for subresource in resource.resources: + if subresource.type == "meta": + if not override: + raise MappingError( + "A type='meta' resource already exists in the first 'result' resource." + ) + else: + logging.info("Overriding existing type='meta' resource.") + break + mivot_resource = Resource() + mivot_resource.type = "meta" + mivot_resource.mivot_block = MivotBlock(self._mivot_block) + resource.resources.append(mivot_resource) diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py new file mode 100644 index 000000000..a8e4b06c4 --- /dev/null +++ b/pyvo/mivot/writer/instance.py @@ -0,0 +1,160 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +MivotInstance is a simple API for building MIVOT instances step by step. +""" + +from pyvo.utils.prototype import prototype_feature +from pyvo.mivot.utils.exceptions import MappingError + +__all__ = ["MivotInstance"] + + +@prototype_feature("MIVOT") +class MivotInstance: + """ + API for building elements of MIVOT annotation step by step. + + This class provides methods for incremental construction of a MIVOT instance. + It builds elements that can contain , , and . + Support for elements is not yet implemented. + + The main features are: + + - Model-agnostic: The implementation is independent of any specific data model. + - Syntax validation: Ensures basic MIVOT syntax rules are followed. + - Context-agnostic: Ignores context-dependent syntax rules. + + attributes + ---------- + _dmtype : string + Instance type (class VO-DML ID) + _dmrole : string + Role played by the instance in the context where it is used + (given by the VO-DML serialization of the model) + _dmid : string + Free identifier of the instance + + """ + def __init__(self, dmtype=None, *, dmrole=None, dmid=None): + """ + Parameters + ---------- + dmtype : str + dmtype of the INSTANCE (mandatory) + dmrole : str, optional + dmrole of the INSTANCE + dmid : str, optional + dmid of the INSTANCE + + Raises + ------ + MappingError + If ``dmtype`` is not provided + """ + if not dmtype: + raise MappingError("Cannot build an instance without dmtype") + self._dmtype = dmtype + self._dmrole = dmrole + self._dmid = dmid + self._content = [] + + def add_attribute(self, dmtype=None, dmrole=None, *, ref=None, value=None, unit=None): + """ + Add an element to the instance. + + Parameters + ---------- + dmtype : str + dmtype of the ATTRIBUTE (mandatory) + dmrole : str + dmrole of the ATTRIBUTE (mandatory) + ref : str, optional + ID of the column to set the attribute value + value : str, optional + Default value of the attribute + unit : str, optional + Unit of the attribute + + Raises + ------ + MappingError + If ``dmtype`` or ``dmrole`` is not provided, or if both ``ref`` and ``value`` are not defined + """ + if not dmtype: + raise MappingError("Cannot add an attribute without dmtype") + if not dmrole: + raise MappingError("Cannot add an attribute without dmrole") + if not ref and not value: + raise MappingError("Cannot add an attribute without ref or value") + + xml_string = f' element to the instance. + + Parameters + ---------- + dmrole : str + dmrole of the REFERENCE (mandatory) + dmref : str + dmref of the REFERENCE (mandatory) + + Raises + ------ + MappingError + If ``dmrole`` or ``dmref`` is not provided + """ + if not dmref: + raise MappingError("Cannot add a reference without dmref") + if not dmrole: + raise MappingError("Cannot add a reference without dmrole") + + xml_string = f'' + self._content.append(xml_string) + + def add_instance(self, mivot_instance): + """ + Add a nested element to the instance. + + Parameters + ---------- + mivot_instance : MivotInstance + INSTANCE to be added + + Raises + ------ + MappingError + If ``mivot_instance`` is not of type ``MivotInstance`` + """ + if not isinstance(mivot_instance, MivotInstance): + raise MappingError("Instance added must be of type MivotInstance") + self._content.append(mivot_instance.xml_string()) + + def xml_string(self): + """ + Build and serialize the element as a string. + + Returns + ------- + str + The string representation of the element + """ + xml_string = f' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file