From 4fc872720295bbd79326fad40207c8790c9363ae Mon Sep 17 00:00:00 2001 From: dschristianson Date: Wed, 19 Jul 2023 11:49:32 -0700 Subject: [PATCH] DataSourceModelAccess.retrieve and related updates Closes #168 Details: - Add translate logic to DataSourceModelAccess.retrieve method - Removed QueryById query class and added `id` property to QueryBase - Added MonitoringFeatureModelAccess.retrieve class to validate specification of `id' - Updated translation logic of query prefixed_fields and validation of prefixed_fields - Added tests to explicitly test the retrieve options. - Removed use of basin3d_plugin_access decorator and made it private --- basin3d/core/plugin.py | 2 +- basin3d/core/schema/query.py | 12 +- basin3d/core/synthesis.py | 140 +++++--- basin3d/core/translate.py | 64 ++-- basin3d/plugins/usgs.py | 38 ++- basin3d/synthesis.py | 12 +- tests/test_core_plugins.py | 12 +- tests/test_core_schema.py | 10 +- tests/test_core_synthesis.py | 21 +- tests/test_core_translate.py | 4 +- tests/test_synthesis.py | 105 +++--- tests/testplugins/alpha.py | 471 ++++++++++++++------------- tests/testplugins/no_plugin_views.py | 2 +- tests/testplugins/plugin_error.py | 89 ++--- 14 files changed, 548 insertions(+), 434 deletions(-) diff --git a/basin3d/core/plugin.py b/basin3d/core/plugin.py index cad7ad7..96efb36 100644 --- a/basin3d/core/plugin.py +++ b/basin3d/core/plugin.py @@ -91,7 +91,7 @@ def basin3d_plugin(cls): return cls -def basin3d_plugin_access(plugin_class, synthesis_model_class, access_type): +def _basin3d_plugin_access(plugin_class, synthesis_model_class, access_type): """Decorator for registering model access""" def _inner(func): diff --git a/basin3d/core/schema/query.py b/basin3d/core/schema/query.py index 26c4436..72dae4e 100644 --- a/basin3d/core/schema/query.py +++ b/basin3d/core/schema/query.py @@ -36,6 +36,9 @@ class QueryBase(BaseModel): datasource: Optional[List[str]] = Field(title="Datasource Identifiers", description="List of datasource identifiers to query by.") + + id: Optional[str] = Field(title="Identifier", description="The unique identifier for the desired object") + is_valid_translated_query: Union[None, bool] = Field(default=None, title="Valid translated query", description="Indicates whether the translated query is valid: None = is not translated") @@ -68,14 +71,9 @@ class Config: prefixed_fields: ClassVar[List[str]] = [] -class QueryById(QueryBase): - """Query for a single data object by identifier""" - - id: str = Field(title="Identifier", description="The unique identifier for the desired data object") - - class QueryMonitoringFeature(QueryBase): """Query :class:`basin3d.core.models.MonitoringFeature`""" + # optional but id (QueryBase) is required to query by named monitoring feature feature_type: Optional[FeatureTypeEnum] = Field(title="Feature Type", description="Filter results by the specified feature type.") monitoring_feature: Optional[List[str]] = Field(title="Monitoring Features", @@ -101,7 +99,7 @@ def __init__(self, **data): data[field] = isinstance(data[field], str) and data[field].upper() or data[field] super().__init__(**data) - prefixed_fields: ClassVar[List[str]] = ['monitoring_feature', 'parent_feature'] + prefixed_fields: ClassVar[List[str]] = ['id', 'monitoring_feature', 'parent_feature'] class QueryMeasurementTimeseriesTVP(QueryBase): diff --git a/basin3d/core/synthesis.py b/basin3d/core/synthesis.py index 2bfd8d2..46837a6 100644 --- a/basin3d/core/synthesis.py +++ b/basin3d/core/synthesis.py @@ -19,7 +19,7 @@ from basin3d.core.models import Base, MeasurementTimeseriesTVPObservation, MonitoringFeature from basin3d.core.plugin import DataSourcePluginAccess, DataSourcePluginPoint from basin3d.core.schema.enum import MessageLevelEnum, AggregationDurationEnum -from basin3d.core.schema.query import QueryBase, QueryById, QueryMeasurementTimeseriesTVP, \ +from basin3d.core.schema.query import QueryBase, QueryMeasurementTimeseriesTVP, \ QueryMonitoringFeature, SynthesisMessage, SynthesisResponse from basin3d.core.translate import translate_query @@ -33,7 +33,7 @@ class MonitorMixin(object): def log(self, message: str, level: Optional[MessageLevelEnum] = None, where: Optional[List] = None) -> Optional[SynthesisMessage]: """ - Add a synthesis message to the synthesis respoonse + Add a synthesis message to the synthesis response :param message: The message :param level: The message level :param where: Where the message is from @@ -188,7 +188,7 @@ def __next__(self) -> Base: def log(self, message: str, level: Optional[MessageLevelEnum] = None, where: Optional[List] = None): # type: ignore[override] """ - Add a synthesis message to the synthesis respoonse + Add a synthesis message to the synthesis response :param message: The message :param level: The message level :return: None @@ -237,45 +237,74 @@ def list(self, query: QueryBase) -> DataSourceModelIterator: return DataSourceModelIterator(query, self) @monitor.ctx_synthesis - def retrieve(self, query: QueryById) -> SynthesisResponse: + def retrieve(self, query: QueryBase, **kwargs) -> SynthesisResponse: """ Retrieve a single synthesized value :param query: The query for this request + :param kwargs: messages, list of messages to be returned in the SynthesisResponse """ - messages = [] - if query.id: - - # split the datasource id prefix from the primary key - id_list = query.id.split("-") - datasource = None - try: - plugin = self.plugins[id_list[0]] - datasource = plugin.get_datasource() - except KeyError: - pass - - if datasource: - datasource_pk = query.id.replace("{}-".format(id_list[0]), - "", 1) # The datasource id prefix needs to be removed - - plugin_views = plugin.get_plugin_access() - monitor.set_ctx_basin3d_where([plugin.get_datasource().id, self.synthesis_model.__name__]) - if self.synthesis_model in plugin_views: - synthesized_query: QueryById = query.copy() - synthesized_query.id = datasource_pk - synthesized_query.datasource = [datasource.id] - obj: Base = plugin_views[self.synthesis_model].get(query=synthesized_query) - return SynthesisResponse(query=query, data=obj) # type: ignore[call-arg] + datasource_id: Optional[str] = None + datasource = None + messages: List[Optional[SynthesisMessage]] = [] + + if 'messages' in kwargs and kwargs.get('messages'): + msg = kwargs.get('messages') + if isinstance(msg, List) and isinstance(msg[0], SynthesisMessage): + messages.extend(msg) + else: + messages.append(self.log('Message mis-configured.', MessageLevelEnum.WARN)) + + # if the datasource is in the query, utilize it straight up + if query.datasource: + datasource_id = query.datasource[0] + + # otherwise, get from a prefixed field. + # Not tying specifically to id at this point to enable different retrieve approaches from future classes. + elif query.prefixed_fields: + value = None + for prefixed_field in query.prefixed_fields: + attr_value = getattr(query, prefixed_field) + if attr_value and isinstance(attr_value, str): + value = attr_value + break + elif attr_value and isinstance(attr_value, list): + value = attr_value[0] + break + + if value and isinstance(value, str): + datasource_id = value.split("-")[0] + + try: + plugin = self.plugins[datasource_id] + datasource = plugin.get_datasource() + except KeyError: + pass + + if datasource: + + plugin_views = plugin.get_plugin_access() + monitor.set_ctx_basin3d_where([plugin.get_datasource().id, self.synthesis_model.__name__]) + if self.synthesis_model in plugin_views and hasattr(plugin_views[self.synthesis_model], 'get'): + + # Now translate the query object + translated_query_params: QueryBase = self.synthesize_query(plugin_views[self.synthesis_model], query) + translated_query_params.datasource = [plugin.get_datasource().id] + + # Get the model access iterator if synthesized query is valid + if translated_query_params.is_valid_translated_query: + item: Optional[Base] = plugin_views[self.synthesis_model].get(query=translated_query_params) else: - messages.append(self.log("Plugin view does not exist", - MessageLevelEnum.WARN, - [plugin.get_datasource().id, self.synthesis_model.__name__], - )) + logger.warning(f'Translated query for datasource {plugin.get_datasource().id} is not valid.') + if item: + return SynthesisResponse(query=query, data=item, messages=messages) # type: ignore[call-arg] else: - messages.append(self.log(f"DataSource not not found for id {query.id}", - MessageLevelEnum.ERROR)) + messages.append(self.log("Plugin view does not exist", MessageLevelEnum.WARN, + [plugin.get_datasource().id, self.synthesis_model.__name__],)) + + else: + messages.append(self.log("DataSource not found for retrieve request", MessageLevelEnum.ERROR)) return SynthesisResponse(query=query, messages=messages) # type: ignore[call-arg] @@ -299,11 +328,13 @@ class MonitoringFeatureAccess(DataSourceModelAccess): * *utc_offset:* float, Coordinate Universal Time offset in hours (offset in hours), e.g., +9 * *url:* url, URL with details for the feature - **Filter** by the following attributes (/?attribute=parameter&attribute=parameter&...) + **Filter** by the following attributes - * *datasource (optional):* a single data source id prefix (e.g ?datasource=`datasource.id_prefix`) + * *datasource (optional):* str, a single data source id prefix + * *id (optional):* str, a single monitoring feature id. Cannot be combined with monitoring_feature + * *parent_feature (optional)*: list, a list of parent feature ids. Plugin must have this functionality enabled. + * *monitoring_feature (optional)*: list, a list of monitoring feature ids. Cannot be combined with id, which will take precedence. - **Restrict fields** with query parameter ‘fields’. (e.g. ?fields=id,name) """ synthesis_model = MonitoringFeature @@ -319,6 +350,34 @@ def synthesize_query(self, plugin_access: DataSourcePluginAccess, query: QueryMo """ return translate_query(plugin_access, query) + def retrieve(self, query: QueryMonitoringFeature) -> SynthesisResponse: + """ + Retrieve the specified Monitoring Feature + + :param query: :class:`basin3d.core.schema.query.QueryMonitoringFeature`, id must be specified; monitoring_feature if specified will be removed. + :return: The synthesized response containing the specified MonitoringFeature if it exists + """ + + # validate that id is specified + if not query.id: + return SynthesisResponse( + query=query, + messages=[self.log('query.id field is missing and is required for monitoring feature request by id.', + MessageLevelEnum.ERROR)]) # type: ignore[call-arg] + + msg = [] + + # remove monitoring_feature specification (i.e., id takes precedence) + if query.monitoring_feature: + mf_text = ', '.join(query.monitoring_feature) + query.monitoring_feature = None + msg.append(self.log(f'Monitoring Feature query has both id {query.id} and monitoring_feature {mf_text} ' + f'specified. Removing monitoring_feature and using id.', MessageLevelEnum.WARN)) + + # retrieve / get method order should be: MonitoringFeatureAccess, DataSourceModelAccess, MonitoringFeatureAccess.get + synthesis_response: SynthesisResponse = super().retrieve(query=query, messages=msg) + return synthesis_response + class MeasurementTimeseriesTVPObservationAccess(DataSourceModelAccess): """ @@ -341,7 +400,7 @@ class MeasurementTimeseriesTVPObservationAccess(DataSourceModelAccess): * *statistic:* list, statistical properties of the observation result (MEAN, MIN, MAX, TOTAL) * *result_quality:* list, quality assessment values contained in the result (VALIDATED, UNVALIDATED, SUSPECTED, REJECTED, ESTIMATED) - **Filter** by the following attributes (?attribute=parameter&attribute=parameter&...): + **Filter** by the following attributes: * *monitoring_feature (required):* List of monitoring_features ids * *observed_property (required):* List of observed property variable ids @@ -351,10 +410,7 @@ class MeasurementTimeseriesTVPObservationAccess(DataSourceModelAccess): * *statistic (optional):* List of statistic options, enum (INSTANT|MEAN|MIN|MAX|TOTAL) * *datasource (optional):* a single data source id prefix (e.g ?datasource=`datasource.id_prefix`) * *result_quality (optional):* enum (VALIDATED|UNVALIDATED|SUSPECTED|REJECTED|ESTIMATED) - * *sampling_medium (optional):* ADD here -- probably should be enum - - **Restrict fields** with query parameter ‘fields’. (e.g. ?fields=id,name) - + * *sampling_medium (optional):* enum (SOLID_PHASE|WATER|GAS|OTHER) """ synthesis_model = MeasurementTimeseriesTVPObservation diff --git a/basin3d/core/translate.py b/basin3d/core/translate.py index 4438732..ae24b87 100644 --- a/basin3d/core/translate.py +++ b/basin3d/core/translate.py @@ -16,7 +16,7 @@ from basin3d.core import monitor from basin3d.core.schema.enum import MAPPING_DELIMITER, NO_MAPPING_TEXT -from basin3d.core.schema.query import QueryBase, QueryById, QueryMeasurementTimeseriesTVP, QueryMonitoringFeature +from basin3d.core.schema.query import QueryBase, QueryMeasurementTimeseriesTVP, QueryMonitoringFeature logger = monitor.get_logger(__name__) @@ -114,26 +114,30 @@ def _is_translated_query_valid(datasource_id, query, translated_query) -> Option :param translated_query: the translated query :return: boolean (True = valid translated query, False = invalid translated query) or None (translated query could not be assessed) """ - # loop thru kwargs - for attr in query.mapped_fields: - translated_attr_value = getattr(translated_query, attr) - b3d_attr_value = getattr(query, attr) - if isinstance(b3d_attr_value, list): - b3d_attr_value = ', '.join(b3d_attr_value) - if translated_attr_value and isinstance(translated_attr_value, list): - # if list and all of list == NOT_SUPPORTED, False - if all([x == NO_MAPPING_TEXT for x in translated_attr_value]): - logger.warning(f'Translated query for datasource {datasource_id} is invalid. No vocabulary found for attribute {attr} with values: {b3d_attr_value}.') - return False - elif translated_attr_value and isinstance(translated_attr_value, str): - # if single NOT_SUPPORTED, False - if translated_attr_value == NO_MAPPING_TEXT: - logger.warning(f'Translated query for datasource {datasource_id} is invalid. No vocabulary found for attribute {attr} with value: {b3d_attr_value}.') - return False - elif translated_attr_value: - logger.warning( - f'Translated query for datasource {datasource_id} cannot be assessed. Translated value for {attr} is not expected type.') - return None + for field_type, field_list in zip(['mapped', 'prefixed'], [query.mapped_fields, query.prefixed_fields]): + # loop thru kwargs + for attr in field_list: + translated_attr_value = getattr(translated_query, attr) + b3d_attr_value = getattr(query, attr) + msg_prefix = '' + if field_type == 'mapped': + msg_prefix = 'No vocabulary found for attribute {attr} with values: {b3d_attr_value}.' + if isinstance(b3d_attr_value, list): + b3d_attr_value = ', '.join(b3d_attr_value) + if translated_attr_value and isinstance(translated_attr_value, list): + # if list and all of list == NOT_SUPPORTED, False + if all([x == NO_MAPPING_TEXT for x in translated_attr_value]): + logger.warning(f'Translated query for datasource {datasource_id} is invalid.{msg_prefix}') + return False + elif translated_attr_value and isinstance(translated_attr_value, str): + # if single NOT_SUPPORTED, False + if translated_attr_value == NO_MAPPING_TEXT: + logger.warning(f'Translated query for datasource {datasource_id} is invalid.{msg_prefix}') + return False + elif translated_attr_value: + logger.warning( + f'Translated query for datasource {datasource_id} cannot be assessed. Translated value for {attr} is not expected type.') + return None return True @@ -175,7 +179,7 @@ def _order_mapped_fields(plugin_access, query_mapped_fields): return query_mapped_fields_ordered -def _translate_mapped_query_attrs(plugin_access, query: Union[QueryMeasurementTimeseriesTVP, QueryMonitoringFeature, QueryById]) -> QueryBase: +def _translate_mapped_query_attrs(plugin_access, query: Union[QueryMeasurementTimeseriesTVP, QueryMonitoringFeature]) -> QueryBase: """ Translation functionality """ @@ -214,7 +218,7 @@ def _translate_mapped_query_attrs(plugin_access, query: Union[QueryMeasurementTi return query -def _translate_prefixed_query_attrs(plugin_access, query: Union[QueryMeasurementTimeseriesTVP, QueryMonitoringFeature, QueryById]) -> QueryBase: +def _translate_prefixed_query_attrs(plugin_access, query: Union[QueryMeasurementTimeseriesTVP, QueryMonitoringFeature]) -> QueryBase: """ :param plugin_access: @@ -237,12 +241,18 @@ def extract_id(identifer): for attr in query.prefixed_fields: attr_value = getattr(query, attr) if attr_value: + + # if the value is a string if isinstance(attr_value, str): - attr_value = attr_value.split(",") + translated_value = extract_id(attr_value) + if translated_value == attr_value: + translated_value = NO_MAPPING_TEXT - translated_values = [extract_id(x) for x in attr_value if x.startswith("{}-".format(id_prefix))] + # otherwise assume it is a list + else: + translated_value = [extract_id(x) for x in attr_value if x.startswith("{}-".format(id_prefix))] - setattr(query, attr, translated_values) + setattr(query, attr, translated_value) return query @@ -385,7 +395,7 @@ def translate_attributes(plugin_access, mapped_attrs, **kwargs): return kwargs -def translate_query(plugin_access, query: Union[QueryMeasurementTimeseriesTVP, QueryMonitoringFeature, QueryById]) -> QueryBase: +def translate_query(plugin_access, query: Union[QueryMeasurementTimeseriesTVP, QueryMonitoringFeature]) -> QueryBase: """ Translate BASIN-3D vocabulary specified in a query to the datasource vocabularies defined by :class:`basin3d.core.models.AttributeMapping` objects specified in the datasource plugin. diff --git a/basin3d/plugins/usgs.py b/basin3d/plugins/usgs.py index 8de4acb..81b2be6 100644 --- a/basin3d/plugins/usgs.py +++ b/basin3d/plugins/usgs.py @@ -53,9 +53,8 @@ import requests -# Get an instance of a logger from basin3d.core.schema.enum import FeatureTypeEnum, AggregationDurationEnum -from basin3d.core.schema.query import QueryById, QueryMeasurementTimeseriesTVP, QueryMonitoringFeature +from basin3d.core.schema.query import QueryMeasurementTimeseriesTVP, QueryMonitoringFeature from basin3d.core.access import get_url from basin3d.core.models import AbsoluteCoordinate, AltitudeCoordinate, Coordinate, GeographicCoordinate, \ MeasurementTimeseriesTVPObservation, MonitoringFeature, RelatedSamplingFeature, \ @@ -437,7 +436,7 @@ def get_hydrological_unit_codes(self, synthesis_messages): return usgs_huc_codes.CONTENT - def get(self, query: QueryById): + def get(self, query: QueryMonitoringFeature): """ Get a single Region object ===================== === ===================== @@ -451,25 +450,30 @@ def get(self, query: QueryById): :param query: The query info object :return: a serialized ``MonitoringFeature`` object """ + # query.id will always be a string at this point with validation upstream, thus ignoring the type checking + + monitoring_feature_len = len(query.id) # type: ignore[arg-type] + if not query.feature_type: + if monitoring_feature_len == 2: + query.feature_type = FeatureTypeEnum.REGION + elif monitoring_feature_len == 4: + query.feature_type = FeatureTypeEnum.SUBREGION + elif monitoring_feature_len == 6: + query.feature_type = FeatureTypeEnum.BASIN + elif monitoring_feature_len == 8: + query.feature_type = FeatureTypeEnum.SUBBASIN + else: + query.feature_type = FeatureTypeEnum.POINT - if len(query.id) == 2: - mf_query = QueryMonitoringFeature(monitoring_feature=[query.id], feature_type=FeatureTypeEnum.REGION) - elif len(query.id) == 4: - mf_query = QueryMonitoringFeature(monitoring_feature=[query.id], feature_type=FeatureTypeEnum.SUBREGION) - elif len(query.id) == 6: - mf_query = QueryMonitoringFeature(monitoring_feature=[query.id], feature_type=FeatureTypeEnum.BASIN) - elif len(query.id) == 8: - mf_query = QueryMonitoringFeature(monitoring_feature=[query.id], feature_type=FeatureTypeEnum.SUBBASIN) - else: - mf_query = QueryMonitoringFeature(monitoring_feature=[query.id], feature_type=FeatureTypeEnum.POINT) + query.monitoring_feature = [query.id] # type: ignore[list-item] - for o in self.list(query=mf_query): + for o in self.list(query=query): return o # An 8 character code can also be a point, Try that - if len(query.id) == 8: - for o in self.list(query=QueryMonitoringFeature(monitoring_feature=[query.id], - feature_type=FeatureTypeEnum.POINT)): + if monitoring_feature_len == 8: + query.feature_type = FeatureTypeEnum.POINT + for o in self.list(query=query): return o return None diff --git a/basin3d/synthesis.py b/basin3d/synthesis.py index fc0c86c..fab0cfb 100644 --- a/basin3d/synthesis.py +++ b/basin3d/synthesis.py @@ -35,8 +35,7 @@ from basin3d.core.catalog import CatalogTinyDb from basin3d.core.models import DataSource from basin3d.core.plugin import PluginMount -from basin3d.core.schema.query import QueryById, QueryMeasurementTimeseriesTVP, \ - QueryMonitoringFeature, SynthesisResponse +from basin3d.core.schema.query import QueryMeasurementTimeseriesTVP, QueryMonitoringFeature, SynthesisResponse from basin3d.core.synthesis import DataSourceModelIterator, MeasurementTimeseriesTVPObservationAccess, \ MonitoringFeatureAccess, logger @@ -183,7 +182,7 @@ def attribute_mappings(self, datasource_id=None, attr_type=None, attr_vocab=None """ return self._catalog.find_attribute_mappings(datasource_id=datasource_id, attr_type=attr_type, attr_vocab=attr_vocab, from_basin3d=from_basin3d) - def monitoring_features(self, query: Union[QueryById, QueryMonitoringFeature] = None, **kwargs) -> Union[ + def monitoring_features(self, query: QueryMonitoringFeature = None, **kwargs) -> Union[ DataSourceModelIterator, SynthesisResponse]: """ Search for all USGS monitoring features, USGS points by parent monitoring features, or look for a single monitoring feature by id. @@ -246,13 +245,10 @@ def monitoring_features(self, query: Union[QueryById, QueryMonitoringFeature] = or a :class:`~basin3d.core.synthesis.DataSourceModelIterator` for multple :class:`MonitoringFeature` objects. """ if not query: - if "id" in kwargs and isinstance(kwargs["id"], str): - query = QueryById(**kwargs) - else: - query = QueryMonitoringFeature(**kwargs) + query = QueryMonitoringFeature(**kwargs) # Search for single or list? - if isinstance(query, QueryById): + if query.id: # mypy casts are only used as hints for the type checker, # and they don’t perform a runtime type check. return cast(SynthesisResponse, self._monitoring_feature_access.retrieve(query=query)) diff --git a/tests/test_core_plugins.py b/tests/test_core_plugins.py index ca13ba8..4486438 100644 --- a/tests/test_core_plugins.py +++ b/tests/test_core_plugins.py @@ -35,9 +35,9 @@ def test_plugin_access_objects(): basin3d.core.models.MonitoringFeature] assert plugin_access_objects[ - basin3d.core.models.MonitoringFeature].__class__.__name__ == 'tests.testplugins.alpha.AlphaMonitoringFeatureAccess' + basin3d.core.models.MonitoringFeature].__class__.__name__ == 'AlphaMonitoringFeatureAccess' assert plugin_access_objects[basin3d.core.models.MeasurementTimeseriesTVPObservation].__class__.__name__ == \ - 'tests.testplugins.alpha.AlphaMeasurementTimeseriesTVPObservationAccess' + 'AlphaMeasurementTimeseriesTVPObservationAccess' def test_plugin_incomplete(): @@ -99,13 +99,13 @@ def test_get_feature_type(feature_name, return_format, result): {'start_date': '2019-10-01', 'observed_property': ["Ag"], 'monitoring_feature': ['E-region']}), # AlphaSource.MF ("tests.testplugins.alpha.AlphaSourcePlugin", - [{'level': 'ERROR', 'msg': 'DataSource not not found for id E-1234', 'where': None}], + [{'level': 'ERROR', 'msg': 'DataSource not found for retrieve request', 'where': None}], "monitoring_features", {"id": 'E-1234'}), - # USGS.MR + # USGS.MF ("basin3d.plugins.usgs.USGSDataSourcePlugin", - [{'level': 'ERROR', 'msg': 'DataSource not not found for id E-1234', 'where': None}], + [{'level': 'ERROR', 'msg': 'DataSource not found for retrieve request', 'where': None}], "monitoring_features", {"id": 'E-1234'}), # NoPluginViews.TVP @@ -124,7 +124,7 @@ def test_get_feature_type(feature_name, return_format, result): "measurement_timeseries_tvp_observations", {'start_date': '2019-10-01', 'observed_property': ["Ag"], 'monitoring_feature': ['A-region']}), ], - ids=["ErrorSource.MF", "ErrorSource.TVP", "AlphaSource.MF", "USGS.MR", "NoPluginViews.TVP", "NoPluginViews.MF", "AlphaSource.TVP"]) + ids=["ErrorSource.MF", "ErrorSource.TVP", "AlphaSource.MF", "USGS.MF", "NoPluginViews.TVP", "NoPluginViews.MF", "AlphaSource.TVP"]) def test_plugin_exceptions(plugin, messages, synthesis_call, synthesis_args): """Test that basin3d handles unexpected exceptions""" diff --git a/tests/test_core_schema.py b/tests/test_core_schema.py index f021b5a..cbf3726 100644 --- a/tests/test_core_schema.py +++ b/tests/test_core_schema.py @@ -15,16 +15,18 @@ "parent_feature": ['moo']}, False), ({"feature_type": 'POINT'}, False), ({"feature_type": 'FOO'}, True), - ({"datasource": 'FOO'}, False)], + ({"datasource": 'FOO'}, False), + ({"id": 'foo'}, False), + ({"id": ['foo']}, True)], ids=["empty", "valid1", "valid2", "feature-type-valid", "feature-type-invalid", - "datasource-invalid"]) + "datasource-invalid", "id-valid", "id-invalid"]) def test_query_monitoring_feature(params, error): """Test the monitoring feature query dataclass""" if not error: query = QueryMonitoringFeature(**params) for p in ["datasource", "monitoring_feature", "parent_feature", - "feature_type"]: + "feature_type", "id"]: if p not in params: assert getattr(query, p) is None @@ -98,7 +100,7 @@ def test_query_measurement_timeseries_tvp(params, error): query = QueryMeasurementTimeseriesTVP(**params) for p in ["datasource", "aggregation_duration", "monitoring_feature", "observed_property", - "start_date", "end_date", "statistic", "result_quality"]: + "start_date", "end_date", "statistic", "result_quality", "id"]: query_json = json.loads(query.json()) if p not in params: diff --git a/tests/test_core_synthesis.py b/tests/test_core_synthesis.py index ca1b5ef..d05d4da 100644 --- a/tests/test_core_synthesis.py +++ b/tests/test_core_synthesis.py @@ -1,9 +1,9 @@ import pytest -from basin3d.core.models import DataSource +from basin3d.core.models import DataSource, MonitoringFeature from basin3d.core.plugin import DataSourcePluginAccess -from basin3d.core.schema.query import QueryMeasurementTimeseriesTVP -from basin3d.core.synthesis import MeasurementTimeseriesTVPObservationAccess +from basin3d.core.schema.query import QueryMeasurementTimeseriesTVP, QueryMonitoringFeature +from basin3d.core.synthesis import MeasurementTimeseriesTVPObservationAccess, MonitoringFeatureAccess, SynthesisResponse from tests.testplugins import alpha @@ -42,3 +42,18 @@ def test_measurement_timeseries_TVP_observation_access_synthesize_query(input_qu result = alpha_access.synthesize_query(alpha_plugin_access, query) assert result.aggregation_duration == expected_result + + +def test_monitoring_feature_retrieve_no_id(): + from basin3d.core.catalog import CatalogTinyDb + catalog = CatalogTinyDb() + catalog.initialize([p(catalog) for p in [alpha.AlphaSourcePlugin]]) + alpha_ds = DataSource(id='Alpha', name='Alpha', id_prefix='A', location='https://asource.foo/') + # cheating and assigning a datasource to the plugin field as it won't be used + alpha_access = MonitoringFeatureAccess({'A': alpha_ds}, catalog) + + result = alpha_access.retrieve(query=QueryMonitoringFeature()) + + assert isinstance(result, SynthesisResponse) + assert result.data is None + assert result.messages[0].msg == 'query.id field is missing and is required for monitoring feature request by id.' diff --git a/tests/test_core_translate.py b/tests/test_core_translate.py index ee62440..e0fe8b6 100644 --- a/tests/test_core_translate.py +++ b/tests/test_core_translate.py @@ -200,8 +200,8 @@ def test_translator_is_translated_query_valid(query, set_translated_attr, expect {'monitoring_feature': ['9237', '00000']}), (QueryMeasurementTimeseriesTVP(monitoring_feature=['F-9237'], observed_property=['Ag'], start_date='2019-01-01'), {'monitoring_feature': []}), - (QueryMonitoringFeature(monitoring_feature=['A-9237', 'R-8e3838'], parent_feature=['A-00000']), - {'monitoring_feature': ['9237'], 'parent_feature': ['00000']}), + (QueryMonitoringFeature(monitoring_feature=['A-9237', 'R-8e3838'], parent_feature=['A-00000'], id='A-345aa'), + {'monitoring_feature': ['9237'], 'parent_feature': ['00000'], 'id': '345aa'}), ], ids=["single", "multiple", "none", "monitoring_feature_query"]) def test_translator_prefixed_query_attrs(alpha_plugin_access, query, set_translated_attr): diff --git a/tests/test_synthesis.py b/tests/test_synthesis.py index 16dd3a4..4ce34cd 100644 --- a/tests/test_synthesis.py +++ b/tests/test_synthesis.py @@ -1,12 +1,11 @@ import datetime -import logging import pytest import basin3d from basin3d.core.catalog import CatalogException from basin3d.core.schema.query import QueryMeasurementTimeseriesTVP, QueryMonitoringFeature from basin3d.core.synthesis import DataSourceModelIterator -from basin3d.synthesis import register, SynthesisException +from basin3d.synthesis import register, SynthesisException, SynthesisResponse def test_register_error(monkeypatch): @@ -58,58 +57,76 @@ def test_register(): assert datasources[1].location == 'https://asource.foo/' -@pytest.mark.parametrize("query", [{"id": "A-123"}, {"id": "A-123", "feature_type": "region"}], - ids=['not-found', 'too-many-for-id']) -def test_monitoring_feature_not_found(query): - """Test not found """ +@pytest.mark.parametrize("query, expected_monitoring_feature_id, msg", + [({"id": "A-123"}, None, None), + ({"id": "A-1", "feature_type": "region"}, None, None), + ({"id": "A-1", 'datasource': 'A'}, "A-1", None), + ({"id": "A-1"}, "A-1", None), + ({"id": "A-1", "monitoring_feature": "A-123"}, "A-1", + 'Monitoring Feature query has both id A-1 and monitoring_feature A-123 specified. Removing monitoring_feature and using id.'), + ({"id": "A-123", "monitoring_feature": "A-1"}, None, + 'Monitoring Feature query has both id A-123 and monitoring_feature A-1 specified. Removing monitoring_feature and using id.'), + ], + ids=['not-found', 'wrong-feature-type', 'point-datasource', 'point', 'with-mf-valid', 'with-mf-invalid']) +def test_monitoring_features_by_id(query, expected_monitoring_feature_id, msg): + """Test query of single monitoring feature using id """ synthesizer = register(['tests.testplugins.alpha.AlphaSourcePlugin']) - pytest.raises(Exception, synthesizer.monitoring_features, **query) + result = synthesizer.monitoring_features(**query) + assert isinstance(result, SynthesisResponse) -def test_monitoring_features_found(): - """Test found """ + if expected_monitoring_feature_id: + assert result.data.id == expected_monitoring_feature_id + else: + assert result.data is None + + if msg is not None: + assert result.messages[0].msg == msg + + +def test_monitoring_features(): + """Test monitoring features""" synthesizer = register(['tests.testplugins.alpha.AlphaSourcePlugin']) - monitoring_featurues = synthesizer.monitoring_features() - if isinstance(monitoring_featurues, DataSourceModelIterator): + monitoring_features = synthesizer.monitoring_features() + if isinstance(monitoring_features, DataSourceModelIterator): count = 0 - assert monitoring_featurues.synthesis_response is not None - assert monitoring_featurues.synthesis_response.dict() == {'data': None, - 'messages': [], - 'query': {'datasource': None, - 'feature_type': None, - 'is_valid_translated_query': None, - 'monitoring_feature': None, - 'parent_feature': None}} - assert isinstance(monitoring_featurues.synthesis_response.query, QueryMonitoringFeature) - - for mf in monitoring_featurues: + assert monitoring_features.synthesis_response is not None + assert monitoring_features.synthesis_response.dict() == {'data': None, + 'messages': [], + 'query': {'datasource': None, + 'feature_type': None, + 'id': None, + 'is_valid_translated_query': None, + 'monitoring_feature': None, + 'parent_feature': None}} + assert isinstance(monitoring_features.synthesis_response.query, QueryMonitoringFeature) + + for mf in monitoring_features: count += 1 - assert monitoring_featurues.synthesis_response is not None - assert monitoring_featurues.synthesis_response.dict() == {'data': None, - 'messages': [{'level': 'WARN', - 'msg': 'message1', - 'where': ['Alpha', - 'MonitoringFeature']}, - {'level': 'WARN', - 'msg': 'message2', - 'where': ['Alpha', - 'MonitoringFeature']}, - {'level': 'WARN', - 'msg': 'message3', - 'where': ['Alpha', - 'MonitoringFeature']}], - 'query': {'datasource': None, - 'feature_type': None, - 'is_valid_translated_query': None, - 'monitoring_feature': None, - 'parent_feature': None}} - - assert count == 2 + assert monitoring_features.synthesis_response is not None + assert monitoring_features.synthesis_response.dict() == {'data': None, + 'messages': [{'level': 'WARN', + 'msg': 'message1', + 'where': ['Alpha', 'MonitoringFeature']}, + {'level': 'WARN', + 'msg': 'message2', + 'where': ['Alpha', 'MonitoringFeature']}, + {'level': 'WARN', + 'msg': 'message3', + 'where': ['Alpha', 'MonitoringFeature']}], + 'query': {'datasource': None, + 'feature_type': None, + 'id': None, + 'is_valid_translated_query': None, + 'monitoring_feature': None, + 'parent_feature': None}} + + assert count == 1 else: - assert monitoring_featurues is not None + assert monitoring_features is not None @pytest.mark.parametrize("query", [{"id": "A-123"}, {"id": "A-123", "feature_type": "region"}], diff --git a/tests/testplugins/alpha.py b/tests/testplugins/alpha.py index f0383c5..f40d85f 100644 --- a/tests/testplugins/alpha.py +++ b/tests/testplugins/alpha.py @@ -4,262 +4,277 @@ from basin3d.core.models import AbsoluteCoordinate, AltitudeCoordinate, Coordinate, DepthCoordinate, \ GeographicCoordinate, MeasurementTimeseriesTVPObservation, MonitoringFeature, RelatedSamplingFeature, \ RepresentativeCoordinate, SpatialSamplingShapes, VerticalCoordinate, ResultListTVP -from basin3d.core.plugin import DataSourcePluginPoint, basin3d_plugin, basin3d_plugin_access +from basin3d.core.plugin import DataSourcePluginPoint, basin3d_plugin, DataSourcePluginAccess from basin3d.core.schema.enum import FeatureTypeEnum, TimeFrequencyEnum -from basin3d.core.schema.query import QueryById, QueryMeasurementTimeseriesTVP, QueryMonitoringFeature +from basin3d.core.schema.query import QueryMeasurementTimeseriesTVP, QueryMonitoringFeature logger = logging.getLogger(__name__) -@basin3d_plugin -class AlphaSourcePlugin(DataSourcePluginPoint): - title = 'Alpha Source Plugin' +class AlphaMeasurementTimeseriesTVPObservationAccess(DataSourcePluginAccess): + """ + Alpha Measurement TVP Observation Access + """ + synthesis_model_class = MeasurementTimeseriesTVPObservation - # Question: should we use the FeatureTypeEnum CV directly? - feature_types = ['REGION', 'POINT', 'TREE'] + def list(self, query: QueryMeasurementTimeseriesTVP): + """ Generate the MeasurementTimeseriesTVPObservation - class DataSourceMeta: - """ - This is an internal metadata class for defining additional :class:`~basin3d.models.DataSource` - attributes. - - **Attributes:** - - *id* - unique id short name - - *name* - human friendly name (more descriptive) - - *location* - resource location - - *id_prefix* - id prefix to make model object ids unique across plugins - - *credentials_format* - if the data source requires authentication, this is where the - format of the stored credentials is defined. + Attributes: + - *id:* string, Cs137 MR survey ID + - *observed_property:* string, Cs137MID + - *utc_offset:* float (offset in hours), +9 + - *geographical_group_id:* string, River system ID (Region ID). + - *geographical_group_type* enum (sampling_feature, site, plot, region) + - *results_points:* Array of DataPoint objects """ - # Data Source attributes - location = 'https://asource.foo/' - id = 'Alpha' # unique id for the datasource - id_prefix = 'A' - name = id # Human Friendly Data Source Name + synthesis_messages: List[str] = [] + data: List[Any] = [] + quality: List[Any] = [] + + if query.monitoring_feature == ['region']: + return StopIteration({"message": "FOO"}) + + supported_monitoring_features = [f'{num}' for num in range(1, 5)] + + if not any([loc_id in supported_monitoring_features for loc_id in query.monitoring_feature]): + return StopIteration({"message": "No data from data source matches monitoring features specified."}) + + location_indices = [] + for loc_id in query.monitoring_feature: + if loc_id in supported_monitoring_features: + location_indices.append(int(loc_id.split('-')[-1])) + + from datetime import datetime + for num in range(1, 10): + data.append((datetime(2016, 2, num), num * 0.3454)) + data = [data, data, [], data] + rqe1 = 'VALIDATED' + rqe2 = 'UNVALIDATED' + rqe3 = 'REJECTED' + quality = [[rqe1, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1], + [rqe2, rqe2, rqe2, rqe2, rqe2, rqe2, rqe2, rqe3, rqe3], + [], + [rqe1, rqe2, rqe3, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1]] + qualities = [[rqe1], + [rqe2, rqe3], + [], + [rqe1, rqe2, rqe3]] + observed_property_variables = ["Acetate", "Acetate", "Aluminum", "Al"] + units = ['nm', 'nm', 'mg/L', 'mg/L'] + statistics = ['mean', 'max', 'mean', 'max'] + + for num in location_indices: + observed_property_variable = observed_property_variables[num - 1] + feature_id = f'A-{str(num - 1)}' + if query: + if observed_property_variable not in query.observed_property: + continue + if query.statistic: + if statistics[num - 1] not in query.statistic: + continue + result_value = data[num - 1] + result_value_quality = quality[num - 1] + result_qualities = qualities[num - 1] + if query.result_quality: + filtered_value = [] + filtered_quality = [] + has_filtered_data_points = 0 + + for v, q in zip(result_value, result_value_quality): + if q in query.result_quality: + filtered_value.append(v) + filtered_quality.append(q) + else: + has_filtered_data_points += 1 + + if has_filtered_data_points > 0: + synthesis_messages.append(f'{feature_id} - {observed_property_variable}: {str(has_filtered_data_points)} timestamps did not match data quality query.') + + if len(filtered_value) == 0: + synthesis_messages.append(f'{feature_id} - {observed_property_variable}: No data values matched result_quality query.') + print(f'{feature_id} - {observed_property_variable}') + continue + + result_value = filtered_value + result_value_quality = filtered_quality + if len(result_value_quality) > 0: + result_qualities = list(set(result_value_quality)) + else: + result_qualities = [] + yield MeasurementTimeseriesTVPObservation( + plugin_access=self, + id=num, + observed_property=observed_property_variable, + utc_offset=-8 - num, + feature_of_interest=MonitoringFeature( + plugin_access=self, + id=num, + name="Point Location " + str(num), + description="The point.", + feature_type=FeatureTypeEnum.POINT, + shape=SpatialSamplingShapes.SHAPE_POINT, + coordinates=Coordinate( + absolute=AbsoluteCoordinate( + horizontal_position=GeographicCoordinate( + units=GeographicCoordinate.UNITS_DEC_DEGREES, + latitude=70.4657, longitude=-20.4567), + vertical_extent=AltitudeCoordinate( + datum=AltitudeCoordinate.DATUM_NAVD88, + value=1500, + distance_units=VerticalCoordinate.DISTANCE_UNITS_FEET)), + representative=RepresentativeCoordinate( + vertical_position=DepthCoordinate( + datum=DepthCoordinate.DATUM_LOCAL_SURFACE, + value=-0.5 - num * 0.1, + distance_units=VerticalCoordinate.DISTANCE_UNITS_METERS) + ) + ), + observed_properties=["Ag", "Acetate", "Aluminum", "Al"], + related_sampling_feature_complex=[ + RelatedSamplingFeature(plugin_access=self, + related_sampling_feature="Region1", + related_sampling_feature_type=FeatureTypeEnum.REGION, + role=RelatedSamplingFeature.ROLE_PARENT)] + ), + feature_of_interest_type=FeatureTypeEnum.POINT, + unit_of_measurement=units[num - 1], + aggregation_duration=TimeFrequencyEnum.DAY, + result_quality=result_qualities, + time_reference_position=None, + statistic=statistics[num - 1], + result=ResultListTVP(plugin_access=self, value=result_value, result_quality=result_value_quality) + ) -@basin3d_plugin_access(AlphaSourcePlugin, MeasurementTimeseriesTVPObservation, 'list') -def find_measurement_timeseries_tvp_observations(self, query: QueryMeasurementTimeseriesTVP): - """ Generate the MeasurementTimeseriesTVPObservation + return StopIteration(synthesis_messages) - Attributes: - - *id:* string, Cs137 MR survey ID - - *observed_property:* string, Cs137MID - - *utc_offset:* float (offset in hours), +9 - - *geographical_group_id:* string, River system ID (Region ID). - - *geographical_group_type* enum (sampling_feature, site, plot, region) - - *results_points:* Array of DataPoint objects +class AlphaMonitoringFeatureAccess(DataSourcePluginAccess): """ - synthesis_messages: List[str] = [] - data: List[Any] = [] - quality: List[Any] = [] - - if query.monitoring_feature == ['region']: - return StopIteration({"message": "FOO"}) - - supported_monitoring_features = [f'{num}' for num in range(1, 5)] - - if not any([loc_id in supported_monitoring_features for loc_id in query.monitoring_feature]): - return StopIteration({"message": "No data from data source matches monitoring features specified."}) - - location_indices = [] - for loc_id in query.monitoring_feature: - if loc_id in supported_monitoring_features: - location_indices.append(int(loc_id.split('-')[-1])) - - from datetime import datetime - for num in range(1, 10): - data.append((datetime(2016, 2, num), num * 0.3454)) - data = [data, data, [], data] - rqe1 = 'VALIDATED' - rqe2 = 'UNVALIDATED' - rqe3 = 'REJECTED' - quality = [[rqe1, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1], - [rqe2, rqe2, rqe2, rqe2, rqe2, rqe2, rqe2, rqe3, rqe3], - [], - [rqe1, rqe2, rqe3, rqe1, rqe1, rqe1, rqe1, rqe1, rqe1]] - qualities = [[rqe1], - [rqe2, rqe3], - [], - [rqe1, rqe2, rqe3]] - observed_property_variables = ["Acetate", "Acetate", "Aluminum", "Al"] - units = ['nm', 'nm', 'mg/L', 'mg/L'] - statistics = ['mean', 'max', 'mean', 'max'] - - for num in location_indices: - observed_property_variable = observed_property_variables[num - 1] - feature_id = f'A-{str(num - 1)}' - if query: - if observed_property_variable not in query.observed_property: - continue - if query.statistic: - if statistics[num - 1] not in query.statistic: - continue - result_value = data[num - 1] - result_value_quality = quality[num - 1] - result_qualities = qualities[num - 1] - if query.result_quality: - filtered_value = [] - filtered_quality = [] - has_filtered_data_points = 0 - - for v, q in zip(result_value, result_value_quality): - if q in query.result_quality: - filtered_value.append(v) - filtered_quality.append(q) - else: - has_filtered_data_points += 1 + Alpha Monitoring Feature Access Plugin + """ + synthesis_model_class = MonitoringFeature - if has_filtered_data_points > 0: - synthesis_messages.append(f'{feature_id} - {observed_property_variable}: {str(has_filtered_data_points)} timestamps did not match data quality query.') + def list(self, query: QueryMonitoringFeature): + """ + Get Monitoring Feature Info + """ + assert query - if len(filtered_value) == 0: - synthesis_messages.append(f'{feature_id} - {observed_property_variable}: No data values matched result_quality query.') - print(f'{feature_id} - {observed_property_variable}') - continue + monitoring_feature_list = query.monitoring_feature + feature_type = query.feature_type - result_value = filtered_value - result_value_quality = filtered_quality - if len(result_value_quality) > 0: - result_qualities = list(set(result_value_quality)) - else: - result_qualities = [] + obj_region = self.synthesis_model_class( + plugin_access=self, + id="Region1", + name="AwesomeRegion", + description="This region is really awesome.", + feature_type=FeatureTypeEnum.REGION, + shape=SpatialSamplingShapes.SHAPE_SURFACE, + coordinates=Coordinate(representative=RepresentativeCoordinate( + representative_point=AbsoluteCoordinate( + horizontal_position=GeographicCoordinate( + units=GeographicCoordinate.UNITS_DEC_DEGREES, + latitude=70.4657, longitude=-20.4567), + vertical_extent=AltitudeCoordinate( + datum=AltitudeCoordinate.DATUM_NAVD88, + value=1500, + distance_units=VerticalCoordinate.DISTANCE_UNITS_FEET)), + representative_point_type=RepresentativeCoordinate.REPRESENTATIVE_POINT_TYPE_CENTER_LOCAL_SURFACE) + ) + ) - yield MeasurementTimeseriesTVPObservation( + if monitoring_feature_list and 'Region1' in monitoring_feature_list: + if feature_type == FeatureTypeEnum.REGION: + yield obj_region + elif feature_type == FeatureTypeEnum.REGION: + yield obj_region + + obj_point = self.synthesis_model_class( plugin_access=self, - id=num, - observed_property=observed_property_variable, - utc_offset=-8 - num, - feature_of_interest=MonitoringFeature( - plugin_access=self, - id=num, - name="Point Location " + str(num), - description="The point.", - feature_type=FeatureTypeEnum.POINT, - shape=SpatialSamplingShapes.SHAPE_POINT, - coordinates=Coordinate( - absolute=AbsoluteCoordinate( - horizontal_position=GeographicCoordinate( - units=GeographicCoordinate.UNITS_DEC_DEGREES, - latitude=70.4657, longitude=-20.4567), - vertical_extent=AltitudeCoordinate( - datum=AltitudeCoordinate.DATUM_NAVD88, - value=1500, - distance_units=VerticalCoordinate.DISTANCE_UNITS_FEET)), - representative=RepresentativeCoordinate( - vertical_position=DepthCoordinate( - datum=DepthCoordinate.DATUM_LOCAL_SURFACE, - value=-0.5 - num * 0.1, - distance_units=VerticalCoordinate.DISTANCE_UNITS_METERS) - ) - ), - observed_properties=["Ag", "Acetate", "Aluminum", "Al"], - related_sampling_feature_complex=[ - RelatedSamplingFeature(plugin_access=self, - related_sampling_feature="Region1", - related_sampling_feature_type=FeatureTypeEnum.REGION, - role=RelatedSamplingFeature.ROLE_PARENT)] + id="1", + name="Point Location 1", + description="The first point.", + feature_type=FeatureTypeEnum.POINT, + shape=SpatialSamplingShapes.SHAPE_POINT, + coordinates=Coordinate( + absolute=AbsoluteCoordinate( + horizontal_position=GeographicCoordinate( + units=GeographicCoordinate.UNITS_DEC_DEGREES, + latitude=70.4657, longitude=-20.4567), + vertical_extent=AltitudeCoordinate( + datum=AltitudeCoordinate.DATUM_NAVD88, + value=1500, + distance_units=VerticalCoordinate.DISTANCE_UNITS_FEET)), + representative=RepresentativeCoordinate( + vertical_position=DepthCoordinate( + datum=DepthCoordinate.DATUM_LOCAL_SURFACE, + value=-0.5, + distance_units=VerticalCoordinate.DISTANCE_UNITS_METERS) + ) ), - feature_of_interest_type=FeatureTypeEnum.POINT, - unit_of_measurement=units[num - 1], - aggregation_duration=TimeFrequencyEnum.DAY, - result_quality=result_qualities, - time_reference_position=None, - statistic=statistics[num - 1], - result=ResultListTVP(plugin_access=self, value=result_value, result_quality=result_value_quality) + observed_properties=["Ag", "Acetate"], + related_sampling_feature_complex=[ + RelatedSamplingFeature(plugin_access=self, + related_sampling_feature="Region1", + related_sampling_feature_type=FeatureTypeEnum.REGION, + role=RelatedSamplingFeature.ROLE_PARENT)] ) - return StopIteration(synthesis_messages) + if not feature_type or feature_type == FeatureTypeEnum.POINT: + if monitoring_feature_list and '1' in monitoring_feature_list: + yield obj_point + elif not monitoring_feature_list: + yield obj_point + return StopIteration(['message1', 'message2', 'message3']) -@basin3d_plugin_access(AlphaSourcePlugin, MeasurementTimeseriesTVPObservation, 'get') -def get_measurement_timeseries_tvp_observation(self, query: QueryById): - """ - Get a MeasurementTimeseriesTVPObservation - :param query: - """ - if query: - for s in self.list(): - if s.id.endswith(query.id): + def get(self, query: QueryMonitoringFeature): + + """ + Get a MonitoringFeature + :param query: Monitoring Feature Query + """ + # query.id is a str at this point due to upstream validation, thus the type ignore. + + prefixed_monitoring_feature = f'{self.datasource.id_prefix}-{query.id}' + + # add id to monitoring_feature field + query.monitoring_feature = [query.id] # type: ignore[list-item] + + for s in self.list(query): + if s.id == prefixed_monitoring_feature: return s - return None + return None -@basin3d_plugin_access(AlphaSourcePlugin, MonitoringFeature, 'list') -def list_monitoring_features(self, query: QueryMonitoringFeature): - """ - Get Monitoring Feature Info - """ - assert query - - obj_region = self.synthesis_model_class( - plugin_access=self, - id="Region1", - name="AwesomeRegion", - description="This region is really awesome.", - feature_type=FeatureTypeEnum.REGION, - shape=SpatialSamplingShapes.SHAPE_SURFACE, - coordinates=Coordinate(representative=RepresentativeCoordinate( - representative_point=AbsoluteCoordinate( - horizontal_position=GeographicCoordinate( - units=GeographicCoordinate.UNITS_DEC_DEGREES, - latitude=70.4657, longitude=-20.4567), - vertical_extent=AltitudeCoordinate( - datum=AltitudeCoordinate.DATUM_NAVD88, - value=1500, - distance_units=VerticalCoordinate.DISTANCE_UNITS_FEET)), - representative_point_type=RepresentativeCoordinate.REPRESENTATIVE_POINT_TYPE_CENTER_LOCAL_SURFACE) - ) - ) - - yield obj_region - - obj_point = self.synthesis_model_class( - plugin_access=self, - id="1", - name="Point Location 1", - description="The first point.", - feature_type=FeatureTypeEnum.POINT, - shape=SpatialSamplingShapes.SHAPE_POINT, - coordinates=Coordinate( - absolute=AbsoluteCoordinate( - horizontal_position=GeographicCoordinate( - units=GeographicCoordinate.UNITS_DEC_DEGREES, - latitude=70.4657, longitude=-20.4567), - vertical_extent=AltitudeCoordinate( - datum=AltitudeCoordinate.DATUM_NAVD88, - value=1500, - distance_units=VerticalCoordinate.DISTANCE_UNITS_FEET)), - representative=RepresentativeCoordinate( - vertical_position=DepthCoordinate( - datum=DepthCoordinate.DATUM_LOCAL_SURFACE, - value=-0.5, - distance_units=VerticalCoordinate.DISTANCE_UNITS_METERS) - ) - ), - observed_properties=["Ag", "Acetate"], - related_sampling_feature_complex=[ - RelatedSamplingFeature(plugin_access=self, - related_sampling_feature="Region1", - related_sampling_feature_type=FeatureTypeEnum.REGION, - role=RelatedSamplingFeature.ROLE_PARENT)] - ) - yield obj_point +@basin3d_plugin +class AlphaSourcePlugin(DataSourcePluginPoint): + title = 'Alpha Source Plugin' + plugin_access_classes = (AlphaMeasurementTimeseriesTVPObservationAccess, AlphaMonitoringFeatureAccess) + + # Question: should we use the FeatureTypeEnum CV directly? + feature_types = ['REGION', 'POINT', 'TREE'] - return StopIteration(['message1', 'message2', 'message3']) + class DataSourceMeta: + """ + This is an internal metadata class for defining additional :class:`~basin3d.models.DataSource` + attributes. + **Attributes:** + - *id* - unique id short name + - *name* - human friendly name (more descriptive) + - *location* - resource location + - *id_prefix* - id prefix to make model object ids unique across plugins + - *credentials_format* - if the data source requires authentication, this is where the + format of the stored credentials is defined. -@basin3d_plugin_access(AlphaSourcePlugin, MonitoringFeature, 'get') -def get_monitoring_feature(self, query: QueryById): - """ - Get a MonitoringFeature - :param pk: primary key - """ - if query: - for s in self.list(): - if s.id.endswith(query.id): - return s - return None + """ + # Data Source attributes + location = 'https://asource.foo/' + id = 'Alpha' # unique id for the datasource + id_prefix = 'A' + name = id # Human Friendly Data Source Name diff --git a/tests/testplugins/no_plugin_views.py b/tests/testplugins/no_plugin_views.py index 8e3de83..5f99fcd 100644 --- a/tests/testplugins/no_plugin_views.py +++ b/tests/testplugins/no_plugin_views.py @@ -1,6 +1,6 @@ import logging -from basin3d.core.plugin import DataSourcePluginPoint, basin3d_plugin, basin3d_plugin_access +from basin3d.core.plugin import DataSourcePluginPoint, basin3d_plugin logger = logging.getLogger(__name__) diff --git a/tests/testplugins/plugin_error.py b/tests/testplugins/plugin_error.py index e1a8e7a..a8c0483 100644 --- a/tests/testplugins/plugin_error.py +++ b/tests/testplugins/plugin_error.py @@ -1,15 +1,58 @@ import logging from basin3d.core.models import MeasurementTimeseriesTVPObservation, MonitoringFeature -from basin3d.core.plugin import DataSourcePluginPoint, basin3d_plugin, basin3d_plugin_access -from basin3d.core.schema.query import QueryById, QueryMeasurementTimeseriesTVP, QueryMonitoringFeature +from basin3d.core.plugin import DataSourcePluginPoint, basin3d_plugin, DataSourcePluginAccess +from basin3d.core.schema.query import QueryMeasurementTimeseriesTVP, QueryMonitoringFeature logger = logging.getLogger(__name__) +class ErrorMeasurementTimeseriesTVPObservationAccess(DataSourcePluginAccess): + """ + MeasurementTimeseriesTVPObservation Access class + """ + synthesis_model_class = MeasurementTimeseriesTVPObservation + + def list(self, query: QueryMeasurementTimeseriesTVP): + """ Generate the MeasurementTimeseriesTVPObservation + + Attributes: + - *id:* string, Cs137 MR survey ID + - *observed_property:* string, Cs137MID + - *utc_offset:* float (offset in hours), +9 + - *geographical_group_id:* string, River system ID (Region ID). + - *geographical_group_type* enum (sampling_feature, site, plot, region) + - *results_points:* Array of DataPoint objects + + """ + raise Exception("This is a find_measurement_timeseries_tvp_observations error") + + +class ErrorMonitoringFeatureAccess(DataSourcePluginAccess): + """ + MonitoringFeature Access + """ + synthesis_model_class = MonitoringFeature + + def list(self, query: QueryMonitoringFeature): + """ + Get Monitoring Feature Info + :param query: + """ + raise Exception("This is a list_monitoring_features exception") + + def get(self, query: QueryMonitoringFeature): + """ + Get a MonitoringFeature + :param query: + """ + raise Exception("This is a get_monitoring_feature exception") + + @basin3d_plugin class ErrorSourcePlugin(DataSourcePluginPoint): title = 'Error Source Plugin' + plugin_access_classes = (ErrorMonitoringFeatureAccess, ErrorMeasurementTimeseriesTVPObservationAccess) # Question: should we use the FeatureTypeEnum CV directly? feature_types = ['REGION', 'POINT', 'TREE'] @@ -33,45 +76,3 @@ class DataSourceMeta: id = 'Error' # unique id for the datasource id_prefix = 'E' name = id # Human Friendly Data Source Name - - -@basin3d_plugin_access(ErrorSourcePlugin, MeasurementTimeseriesTVPObservation, 'list') -def find_measurement_timeseries_tvp_observations(self, query: QueryMeasurementTimeseriesTVP): - """ Generate the MeasurementTimeseriesTVPObservation - - Attributes: - - *id:* string, Cs137 MR survey ID - - *observed_property:* string, Cs137MID - - *utc_offset:* float (offset in hours), +9 - - *geographical_group_id:* string, River system ID (Region ID). - - *geographical_group_type* enum (sampling_feature, site, plot, region) - - *results_points:* Array of DataPoint objects - - """ - raise Exception("This is a find_measurement_timeseries_tvp_observations error") - - -@basin3d_plugin_access(ErrorSourcePlugin, MeasurementTimeseriesTVPObservation, 'get') -def get_measurement_timeseries_tvp_observation(self, query: QueryById): - """ - Get a MeasurementTimeseriesTVPObservation - :param query: - """ - raise Exception("This is a get_measurement_timeseries_tvp_observation error") - - -@basin3d_plugin_access(ErrorSourcePlugin, MonitoringFeature, 'list') -def list_monitoring_features(self, query: QueryMonitoringFeature): - """ - Get Monitoring Feature Info - """ - raise Exception("This is a list_monitoring_features exception") - - -@basin3d_plugin_access(ErrorSourcePlugin, MonitoringFeature, 'get') -def get_monitoring_feature(self, query: QueryById): - """ - Get a MonitoringFeature - :param pk: primary key - """ - raise Exception("This is a get_monitoring_feature exception")