From dfc872247fd2823151c02bb11e46deb0ca5572dd Mon Sep 17 00:00:00 2001 From: Marwa Date: Wed, 24 Jul 2024 14:13:27 +0200 Subject: [PATCH 01/16] test: update v2 relic keyword --- tests/models/test_ngsi_ld_context.py | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index 2dfd873a..e010a040 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -164,18 +164,18 @@ def setUp(self) -> None: } } - def test_cb_attribute(self) -> None: + def test_cb_property(self) -> None: """ - Test context attribute models + Test context property models Returns: None """ - attr = ContextProperty(**{'value': "20"}) - self.assertIsInstance(attr.value, str) - attr = ContextProperty(**{'value': 20.53}) - self.assertIsInstance(attr.value, float) - attr = ContextProperty(**{'value': 20}) - self.assertIsInstance(attr.value, int) + prop = ContextProperty(**{'value': "20"}) + self.assertIsInstance(prop.value, str) + prop = ContextProperty(**{'value': 20.53}) + self.assertIsInstance(prop.value, float) + prop = ContextProperty(**{'value': 20}) + self.assertIsInstance(prop.value, int) def test_entity_id(self) -> None: with self.assertRaises(ValidationError): @@ -235,33 +235,33 @@ def test_get_properties(self): entity = ContextLDEntity(id="urn:ngsi-ld:test", type="Tester") properties = [ - NamedContextProperty(name="attr1"), - NamedContextProperty(name="attr2"), + NamedContextProperty(name="prop1"), + NamedContextProperty(name="prop2"), ] entity.add_properties(properties) self.assertEqual(entity.get_properties(response_format="list"), properties) - def test_entity_delete_attributes(self): + def test_entity_delete_properties(self): """ - Test the delete_attributes methode + Test the delete_properties method """ - attr = ContextProperty(**{'value': 20, 'type': 'Text'}) - named_attr = NamedContextProperty(**{'name': 'test2', + prop = ContextProperty(**{'value': 20, 'type': 'Text'}) + named_prop = NamedContextProperty(**{'name': 'test2', 'value': 20, 'type': 'Text'}) - attr3 = ContextProperty(**{'value': 20, 'type': 'Text'}) + prop3 = ContextProperty(**{'value': 20, 'type': 'Text'}) entity = ContextLDEntity(id="urn:ngsi-ld:12", type="Test") - entity.add_properties({"test1": attr, "test3": attr3}) - entity.add_properties([named_attr]) + entity.add_properties({"test1": prop, "test3": prop3}) + entity.add_properties([named_prop]) - entity.delete_properties({"test1": attr}) + entity.delete_properties({"test1": prop}) self.assertEqual(set([_prop.name for _prop in entity.get_properties()]), {"test2", "test3"}) - entity.delete_properties([named_attr]) + entity.delete_properties([named_prop]) self.assertEqual(set([_prop.name for _prop in entity.get_properties()]), {"test3"}) From 0145681d38c842eb44cb6d2c2ba1680cfc508604 Mon Sep 17 00:00:00 2001 From: Marwa Date: Tue, 3 Sep 2024 14:51:03 +0200 Subject: [PATCH 02/16] tests: adapt property validation and unit tests Subproperties are now checked recursively to deal with nested properties. But list properties apparently cause errors --- filip/models/ngsi_ld/context.py | 59 ++++++++++++++++++----- tests/models/test_ngsi_ld_context.py | 72 +++++++++++++++++++++++----- 2 files changed, 107 insertions(+), 24 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 4ba897c7..f96e7994 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -2,10 +2,11 @@ NGSI LD models for context broker interaction """ import logging -from typing import Any, List, Dict, Union, Optional +from typing import Any, List, Dict, Union, Optional +from typing_extensions import Self from aenum import Enum -from pydantic import field_validator, ConfigDict, BaseModel, Field +from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field from filip.models.ngsi_v2 import ContextEntity from filip.utils.validators import FiwareRegex, \ validate_fiware_datatype_string_protect, validate_fiware_standard_regex @@ -503,6 +504,7 @@ def __init__(self, super().__init__(id=id, type=type, **data) # TODO we should distinguish between context relationship + # TODO is "validate_attributes" still relevant for LD entities? @classmethod def _validate_attributes(cls, data: Dict): fields = set([field.validation_alias for (_, field) in cls.model_fields.items()] + @@ -538,17 +540,48 @@ def _validate_id(cls, id: str): if not id.startswith("urn:ngsi-ld:"): raise ValueError('Id has to be an URN and starts with "urn:ngsi-ld:"') return id - - @classmethod - def _validate_properties(cls, data: Dict): - attrs = {} - for key, attr in data.items(): - if key not in ContextEntity.model_fields: - if attr["type"] == DataTypeLD.RELATIONSHIP: - attrs[key] = ContextRelationship.model_validate(attr) - else: - attrs[key] = ContextProperty.model_validate(attr) - return attrs + + # @classmethod + # def _validate_properties(cls, data: Dict): + # attrs = {} + # for key, attr in data.items(): + # if key not in ContextEntity.model_fields: + # if attr["type"] == DataTypeLD.RELATIONSHIP: + # attrs[key] = ContextRelationship.model_validate(attr) + # else: + # attrs[key] = ContextProperty.model_validate(attr) + # return attrs + + def _validate_single_property(self, data, validity): + if data is None or isinstance(data, (str, int, float)): + return validity + if isinstance(data, list): + for item in data: + validity = validity and self._validate_single_property(item, validity) + elif isinstance(data, dict): + for key, attr in data.items(): + if key == 'type': + if attr == DataTypeLD.RELATIONSHIP: + ContextRelationship.model_validate(data) + else: + ContextProperty.model_validate(data) + validity = validity and self._validate_single_property(attr, validity) + else: + raise NotImplementedError( + f"The property type ({type(data)}) for {data} is not implemented yet") + return validity + + @model_validator(mode='after') + def _validate_properties(self) -> Self: + model_dump = self.model_dump() + valid = True + for key, attr in model_dump.items(): + if key in ContextEntity.model_fields: + continue + valid = self._validate_single_property(attr, valid) + if not valid: + raise ValueError('Properties not valid') + return self def get_properties(self, response_format: Union[str, PropertyFormat] = diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index e010a040..c1a28d2e 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -136,33 +136,65 @@ def setUp(self) -> None: self.entity_sub_props_dict = { "id": "urn:ngsi-ld:Vehicle:test1243", "type": "Vehicle", - "prop1": { + "Make": { "type": "Property", - "value": 1, - "sub_property": { + "value": "Tesla", + "Model": { "type": "Property", - "value": 10, - "sub_sub_property": { + "value": "Model 3", + "Year": { "type": "Property", - "value": 100 + "value": 2024 } }, - "sub_properties_list": [ + "Warranty": [ { - "sub_prop_1": { - "value": 100, + "Coverage": { + "value": "Premium", "type": "Property" } }, { - "sub_prop_2": { - "value": 200, + "Duration": { + "value": 5, "type": "Property" } } ], } } + self.entity_list_property = { + "id": "urn:ngsi-ld:Vehicle:A4567", + "type": "Vehicle", + "speed": [ + { + "type": "Property", + "value": 55, + "source": { + "type": "Property", + "value": "Speedometer" + }, + "datasetId": "urn:ngsi-ld:Property:speedometerA4567-speed" + }, + { + "type": "Property", + "value": 54.5, + "source": { + "type": "Property", + "value": "GPS" + }, + "datasetId": "urn:ngsi-ld:Property:gpsBxyz123-speed" + } + ], + "@context": [ + { + "Vehicle": "http://example.org/Vehicle", + "speed": "http://example.org/speed", + "source": "http://example.org/hasSource" + }, + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + ] + } def test_cb_property(self) -> None: """ @@ -227,6 +259,24 @@ def test_cb_entity(self) -> None: properties = entity2.get_properties(response_format='list') self.assertIn("new_prop", [prop.name for prop in properties]) + def test_validate_subproperties_dict(self) -> None: + """ + Test the validation of multi-level properties in entities + Returns: + None + """ + entity4 = ContextLDEntity(**self.entity_sub_props_dict) + entity4._validate_properties() + + def test_validate_subproperties_list(self) -> None: + """ + Test the validation of multi-level properties in entities + Returns: + None + """ + entity4 = ContextLDEntity(**self.entity_list_property) + entity4._validate_properties() + def test_get_properties(self): """ Test the get_properties method From 94cfe61b53479c0e8ca1c4fe9a83be29c6e17bc0 Mon Sep 17 00:00:00 2001 From: Marwa Date: Wed, 4 Sep 2024 14:37:29 +0200 Subject: [PATCH 03/16] tests: remove property list example --- filip/models/ngsi_ld/context.py | 40 ++++++++++--------- tests/models/test_ngsi_ld_context.py | 58 ++++++++++++++-------------- 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index f96e7994..ee3b4ec4 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -552,36 +552,40 @@ def _validate_id(cls, id: str): # attrs[key] = ContextProperty.model_validate(attr) # return attrs - def _validate_single_property(self, data, validity): - if data is None or isinstance(data, (str, int, float)): + def _validate_single_property(self, key, data, validity): + if key == 'type': + if data == DataTypeLD.RELATIONSHIP: + ContextRelationship.model_validate(data) + else: + ContextProperty.model_validate(data) + elif data is None or isinstance(data, (str, int, float)): + print('Skipping checking ',data,' because single value') return validity - if isinstance(data, list): - for item in data: - validity = validity and self._validate_single_property(item, validity) + # elif isinstance(data, list): + # for item in data: + # validity = validity and self._validate_single_property(item, validity) elif isinstance(data, dict): - for key, attr in data.items(): - if key == 'type': - if attr == DataTypeLD.RELATIONSHIP: - ContextRelationship.model_validate(data) - else: - ContextProperty.model_validate(data) - validity = validity and self._validate_single_property(attr, validity) + for attr_key, attr in data.items(): + validity = validity and self._validate_single_property(attr_key, attr, validity) else: raise NotImplementedError( f"The property type ({type(data)}) for {data} is not implemented yet") return validity - @model_validator(mode='after') - def _validate_properties(self) -> Self: + def _validate_properties(self): model_dump = self.model_dump() + print('\nModel dump as is:\n',model_dump) valid = True + for entity_key in ContextEntity.model_fields: + model_dump.pop(entity_key, None) + print('\nModel dump after removing entity keys:\n',model_dump) for key, attr in model_dump.items(): - if key in ContextEntity.model_fields: - continue - valid = self._validate_single_property(attr, valid) + print('About to check single property ',key,': ',attr) + valid = self._validate_single_property(key, attr, valid) + print('Single property ',attr, ' is valid: ',valid) if not valid: raise ValueError('Properties not valid') - return self + return valid def get_properties(self, response_format: Union[str, PropertyFormat] = diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index c1a28d2e..bdb9c21d 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -163,37 +163,35 @@ def setUp(self) -> None: ], } } - self.entity_list_property = { - "id": "urn:ngsi-ld:Vehicle:A4567", + self.entity_sub_props_dict_wrong = { + "id": "urn:ngsi-ld:Vehicle:test1243", "type": "Vehicle", - "speed": [ - { - "type": "Property", - "value": 55, - "source": { + "Make": { + "type": "NotAProperty", + "value": "Tesla", + "Model": { + "type": "NotAProperty", + "value": "Model 3", + "Year": { "type": "Property", - "value": "Speedometer" - }, - "datasetId": "urn:ngsi-ld:Property:speedometerA4567-speed" + "value": 2024 + } }, - { - "type": "Property", - "value": 54.5, - "source": { - "type": "Property", - "value": "GPS" + "Warranty": [ + { + "Coverage": { + "value": "Premium", + "type": "Property" + } }, - "datasetId": "urn:ngsi-ld:Property:gpsBxyz123-speed" - } - ], - "@context": [ - { - "Vehicle": "http://example.org/Vehicle", - "speed": "http://example.org/speed", - "source": "http://example.org/hasSource" - }, - "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" - ] + { + "Duration": { + "value": 5, + "type": "Property" + } + } + ], + } } def test_cb_property(self) -> None: @@ -268,14 +266,14 @@ def test_validate_subproperties_dict(self) -> None: entity4 = ContextLDEntity(**self.entity_sub_props_dict) entity4._validate_properties() - def test_validate_subproperties_list(self) -> None: + def test_validate_subproperties_dict_wrong(self) -> None: """ Test the validation of multi-level properties in entities Returns: None """ - entity4 = ContextLDEntity(**self.entity_list_property) - entity4._validate_properties() + entity5 = ContextLDEntity(**self.entity_sub_props_dict_wrong) + # entity5._validate_properties() def test_get_properties(self): """ From d0fa3661d7576ab4ca574bd7bc00584dee5d5dec Mon Sep 17 00:00:00 2001 From: Marwa Date: Wed, 4 Sep 2024 15:38:14 +0200 Subject: [PATCH 04/16] refactor: clarify property validator --- filip/models/ngsi_ld/context.py | 92 ++++++++++++++-------------- tests/models/test_ngsi_ld_context.py | 34 ++++------ 2 files changed, 58 insertions(+), 68 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index ee3b4ec4..438b1ba1 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -90,19 +90,13 @@ def check_property_type(cls, value): Returns: value """ - if not value == "Property": - if value == "Relationship": - value == "Relationship" - elif value == "TemporalProperty": - value == "TemporalProperty" - else: - logging.warning(msg='NGSI_LD Properties must have type "Property"') - value = "Property" + valid_property_types = ["Property", "Relationship", "TemporalProperty"] + if value not in valid_property_types: + logging.warning(msg='NGSI_LD Properties must have type "Property"') + logging.warning(msg=f'Changing value from "{value}" to "Property"') + value = "Property" return value - - - class NamedContextProperty(ContextProperty): """ Context properties are properties of context entities. For example, the current speed of a car could be modeled @@ -500,7 +494,9 @@ def __init__(self, type: str, **data): # There is currently no validation for extra fields + print('Data as is:\n',data) data.update(self._validate_attributes(data)) + print('Data after updating:\n',data) super().__init__(id=id, type=type, **data) # TODO we should distinguish between context relationship @@ -509,6 +505,7 @@ def __init__(self, def _validate_attributes(cls, data: Dict): fields = set([field.validation_alias for (_, field) in cls.model_fields.items()] + [field_name for field_name in cls.model_fields]) + print('Fields: ',fields) fields.remove(None) # Initialize the attribute dictionary attrs = {} @@ -516,12 +513,12 @@ def _validate_attributes(cls, data: Dict): # Iterate through the data for key, attr in data.items(): # Check if the keyword is not already present in the fields - if key not in fields: + if key not in fields: # TODO why ignoring all in fields? try: attrs[key] = ContextGeoProperty.model_validate(attr) except ValueError: attrs[key] = ContextProperty.model_validate(attr) - return attrs + return attrs model_config = ConfigDict(extra='allow', validate_default=True, validate_assignment=True) @@ -552,40 +549,41 @@ def _validate_id(cls, id: str): # attrs[key] = ContextProperty.model_validate(attr) # return attrs - def _validate_single_property(self, key, data, validity): - if key == 'type': - if data == DataTypeLD.RELATIONSHIP: - ContextRelationship.model_validate(data) - else: - ContextProperty.model_validate(data) - elif data is None or isinstance(data, (str, int, float)): - print('Skipping checking ',data,' because single value') - return validity - # elif isinstance(data, list): - # for item in data: - # validity = validity and self._validate_single_property(item, validity) - elif isinstance(data, dict): - for attr_key, attr in data.items(): - validity = validity and self._validate_single_property(attr_key, attr, validity) - else: - raise NotImplementedError( - f"The property type ({type(data)}) for {data} is not implemented yet") - return validity - - def _validate_properties(self): - model_dump = self.model_dump() - print('\nModel dump as is:\n',model_dump) - valid = True - for entity_key in ContextEntity.model_fields: - model_dump.pop(entity_key, None) - print('\nModel dump after removing entity keys:\n',model_dump) - for key, attr in model_dump.items(): - print('About to check single property ',key,': ',attr) - valid = self._validate_single_property(key, attr, valid) - print('Single property ',attr, ' is valid: ',valid) - if not valid: - raise ValueError('Properties not valid') - return valid + # def _validate_single_property(self, key, data, validity): + # if key == 'type': + # if data == DataTypeLD.RELATIONSHIP: + # ContextRelationship.model_validate(data) + # else: + # ContextProperty.model_validate(data) + # elif data is None or isinstance(data, (str, int, float)): + # print('Skipping checking ',data,' because single value') + # return validity + # # elif isinstance(data, list): + # # for item in data: + # # validity = validity and self._validate_single_property(item, validity) + # elif isinstance(data, dict): + # for attr_key, attr in data.items(): + # validity = validity and self._validate_single_property(attr_key, attr, validity) + # else: + # raise NotImplementedError( + # f"The property type ({type(data)}) for {data} is not implemented yet") + # return validity + + # @model_validator(mode='before') + # def _validate_properties(self) -> Self: + # model_dump = self.model_dump() + # print('\nModel dump as is:\n',model_dump) + # valid = True + # for entity_key in ContextEntity.model_fields: + # model_dump.pop(entity_key, None) + # print('\nModel dump after removing entity keys:\n',model_dump) + # for key, attr in model_dump.items(): + # print('About to check single property ',key,': ',attr) + # valid = self._validate_single_property(key, attr, valid) + # print('Single property ',attr, ' is valid: ',valid) + # if not valid: + # raise ValueError('Properties not valid') + # return self def get_properties(self, response_format: Union[str, PropertyFormat] = diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index bdb9c21d..eb2359e0 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -145,52 +145,44 @@ def setUp(self) -> None: "Year": { "type": "Property", "value": 2024 - } - }, - "Warranty": [ - { + }, + "Warranty": { "Coverage": { "value": "Premium", "type": "Property" - } - }, - { + }, "Duration": { "value": 5, "type": "Property" } - } - ], + }, + }, } } self.entity_sub_props_dict_wrong = { "id": "urn:ngsi-ld:Vehicle:test1243", "type": "Vehicle", "Make": { - "type": "NotAProperty", + "type": "NotAProperty_level1", "value": "Tesla", "Model": { - "type": "NotAProperty", + "type": "NotAProperty_level2", "value": "Model 3", "Year": { - "type": "Property", + "type": "NotAProperty_level3", "value": 2024 - } - }, - "Warranty": [ - { + }, + "Warranty": { "Coverage": { "value": "Premium", "type": "Property" - } - }, - { + }, "Duration": { "value": 5, "type": "Property" } - } - ], + }, + }, } } From 653daa3ab2d0b1d154c3bdc90521b3f3953316dd Mon Sep 17 00:00:00 2001 From: Marwa Date: Sat, 14 Sep 2024 09:56:25 +0200 Subject: [PATCH 05/16] fix: validate property then check for subproperties --- filip/models/ngsi_ld/context.py | 70 +++++----------------- tests/models/test_ngsi_ld_context.py | 86 +++++++++++----------------- 2 files changed, 49 insertions(+), 107 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 438b1ba1..8210b159 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -494,11 +494,22 @@ def __init__(self, type: str, **data): # There is currently no validation for extra fields - print('Data as is:\n',data) data.update(self._validate_attributes(data)) - print('Data after updating:\n',data) super().__init__(id=id, type=type, **data) + @classmethod + def _validate_single_property(cls, key, attr): + subattrs = {} + try: + subattrs[key] = ContextGeoProperty.model_validate(attr) + except ValueError: + subattrs[key] = ContextProperty.model_validate(attr) + # Check if there are subproperties and call this validate method recursively + for subkey, subattr in attr.items(): + if isinstance(subattr, dict): + subattrs[subkey] = cls._validate_single_property(subkey, subattr) + return subattrs + # TODO we should distinguish between context relationship # TODO is "validate_attributes" still relevant for LD entities? @classmethod @@ -509,15 +520,11 @@ def _validate_attributes(cls, data: Dict): fields.remove(None) # Initialize the attribute dictionary attrs = {} - # Iterate through the data for key, attr in data.items(): # Check if the keyword is not already present in the fields - if key not in fields: # TODO why ignoring all in fields? - try: - attrs[key] = ContextGeoProperty.model_validate(attr) - except ValueError: - attrs[key] = ContextProperty.model_validate(attr) + if key not in fields: # TODO why ignoring all in fields? + attrs[key] = cls._validate_single_property(key, attr) return attrs model_config = ConfigDict(extra='allow', validate_default=True, validate_assignment=True) @@ -537,53 +544,6 @@ def _validate_id(cls, id: str): if not id.startswith("urn:ngsi-ld:"): raise ValueError('Id has to be an URN and starts with "urn:ngsi-ld:"') return id - - # @classmethod - # def _validate_properties(cls, data: Dict): - # attrs = {} - # for key, attr in data.items(): - # if key not in ContextEntity.model_fields: - # if attr["type"] == DataTypeLD.RELATIONSHIP: - # attrs[key] = ContextRelationship.model_validate(attr) - # else: - # attrs[key] = ContextProperty.model_validate(attr) - # return attrs - - # def _validate_single_property(self, key, data, validity): - # if key == 'type': - # if data == DataTypeLD.RELATIONSHIP: - # ContextRelationship.model_validate(data) - # else: - # ContextProperty.model_validate(data) - # elif data is None or isinstance(data, (str, int, float)): - # print('Skipping checking ',data,' because single value') - # return validity - # # elif isinstance(data, list): - # # for item in data: - # # validity = validity and self._validate_single_property(item, validity) - # elif isinstance(data, dict): - # for attr_key, attr in data.items(): - # validity = validity and self._validate_single_property(attr_key, attr, validity) - # else: - # raise NotImplementedError( - # f"The property type ({type(data)}) for {data} is not implemented yet") - # return validity - - # @model_validator(mode='before') - # def _validate_properties(self) -> Self: - # model_dump = self.model_dump() - # print('\nModel dump as is:\n',model_dump) - # valid = True - # for entity_key in ContextEntity.model_fields: - # model_dump.pop(entity_key, None) - # print('\nModel dump after removing entity keys:\n',model_dump) - # for key, attr in model_dump.items(): - # print('About to check single property ',key,': ',attr) - # valid = self._validate_single_property(key, attr, valid) - # print('Single property ',attr, ' is valid: ',valid) - # if not valid: - # raise ValueError('Properties not valid') - # return self def get_properties(self, response_format: Union[str, PropertyFormat] = diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index eb2359e0..f004cac9 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -133,57 +133,41 @@ def setUp(self) -> None: } } # The entity for testing the nested structure of properties - self.entity_sub_props_dict = { - "id": "urn:ngsi-ld:Vehicle:test1243", - "type": "Vehicle", - "Make": { - "type": "Property", - "value": "Tesla", - "Model": { - "type": "Property", - "value": "Model 3", - "Year": { - "type": "Property", - "value": 2024 - }, - "Warranty": { - "Coverage": { - "value": "Premium", - "type": "Property" - }, - "Duration": { - "value": 5, - "type": "Property" - } - }, - }, - } - } self.entity_sub_props_dict_wrong = { - "id": "urn:ngsi-ld:Vehicle:test1243", - "type": "Vehicle", - "Make": { - "type": "NotAProperty_level1", - "value": "Tesla", - "Model": { - "type": "NotAProperty_level2", - "value": "Model 3", - "Year": { - "type": "NotAProperty_level3", - "value": 2024 - }, - "Warranty": { - "Coverage": { - "value": "Premium", - "type": "Property" - }, - "Duration": { - "value": 5, - "type": "Property" - } - }, + "id": "urn:ngsi-ld:OffStreetParking:Downtown1", + "type": "OffStreetParking", + "name": { + "type": "Property", + "value": "Downtown One" + }, + "availableSpotNumber": { + "type": "Property", + "value": 121, + "observedAt": "2017-07-29T12:05:02Z", + "reliability": { + "type": "NotAProperty", + "value": 0.7 }, - } + "providedBy": { + "type": "NotARelationship", + "object": "urn:ngsi-ld:Camera:C1" + } + }, + "totalSpotNumber": { + "type": "Property", + "value": 200 + }, + "location": { + "type": "GeoProperty", + "value": { + "type": "Point", + "coordinates": [-8.5, 41.2] + } + }, + "@context": [ + "http://example.org/ngsi-ld/latest/parking.jsonld", + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + ] } def test_cb_property(self) -> None: @@ -255,8 +239,7 @@ def test_validate_subproperties_dict(self) -> None: Returns: None """ - entity4 = ContextLDEntity(**self.entity_sub_props_dict) - entity4._validate_properties() + entity4 = ContextLDEntity(**self.entity1_dict) def test_validate_subproperties_dict_wrong(self) -> None: """ @@ -265,7 +248,6 @@ def test_validate_subproperties_dict_wrong(self) -> None: None """ entity5 = ContextLDEntity(**self.entity_sub_props_dict_wrong) - # entity5._validate_properties() def test_get_properties(self): """ From 620539863247fc5fb8e48580581d20ba4706b841 Mon Sep 17 00:00:00 2001 From: Marwa Date: Sat, 14 Sep 2024 09:58:44 +0200 Subject: [PATCH 06/16] chore: remove extra print statement --- filip/models/ngsi_ld/context.py | 1 - 1 file changed, 1 deletion(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 8210b159..567f4661 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -516,7 +516,6 @@ def _validate_single_property(cls, key, attr): def _validate_attributes(cls, data: Dict): fields = set([field.validation_alias for (_, field) in cls.model_fields.items()] + [field_name for field_name in cls.model_fields]) - print('Fields: ',fields) fields.remove(None) # Initialize the attribute dictionary attrs = {} From 9ec2987e9ef79db343f8c5ad69224161cb2dacce Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Tue, 17 Sep 2024 12:06:27 +0200 Subject: [PATCH 07/16] chore: change behavior of auto adapting type --- filip/models/ngsi_ld/context.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 567f4661..8ca4354a 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -92,9 +92,10 @@ def check_property_type(cls, value): """ valid_property_types = ["Property", "Relationship", "TemporalProperty"] if value not in valid_property_types: - logging.warning(msg='NGSI_LD Properties must have type "Property"') - logging.warning(msg=f'Changing value from "{value}" to "Property"') - value = "Property" + logging.warning( + msg=f'NGSI_LD Properties must have type {valid_property_types}, ' + f'not "{value}"') + raise ValueError return value class NamedContextProperty(ContextProperty): From 9871b45d742e2ca9345438304e4387c73fe75900 Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Tue, 17 Sep 2024 13:50:43 +0200 Subject: [PATCH 08/16] chore: change behavior of auto adapting type --- filip/models/ngsi_ld/context.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 8ca4354a..e13eb4f5 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -92,10 +92,10 @@ def check_property_type(cls, value): """ valid_property_types = ["Property", "Relationship", "TemporalProperty"] if value not in valid_property_types: - logging.warning( - msg=f'NGSI_LD Properties must have type {valid_property_types}, ' - f'not "{value}"') - raise ValueError + msg = f'NGSI_LD Properties must have type {valid_property_types}, ' \ + f'not "{value}"' + logging.warning(msg=msg) + raise ValueError(msg) return value class NamedContextProperty(ContextProperty): From d705acd8ebb40e672f213485ad6e51d22fc0c1fb Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Wed, 18 Sep 2024 17:33:42 +0200 Subject: [PATCH 09/16] feat: implement the rest geoproperty models --- filip/models/ngsi_ld/context.py | 82 ++++++++----------- tests/models/test_ngsi_ld_context.py | 113 ++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 49 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 85d6c34f..760bd13e 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -4,8 +4,11 @@ import logging from typing import Any, List, Dict, Union, Optional +from geojson_pydantic import Point, MultiPoint, LineString, MultiLineString, Polygon, \ + MultiPolygon, GeometryCollection +from typing_extensions import Self from aenum import Enum -from pydantic import field_validator, ConfigDict, BaseModel, Field +from pydantic import field_validator, ConfigDict, BaseModel, Field, model_validator from filip.models.ngsi_v2 import ContextEntity from filip.utils.validators import FiwareRegex, \ validate_fiware_datatype_string_protect, validate_fiware_standard_regex @@ -159,48 +162,31 @@ class ContextGeoPropertyValue(BaseModel): """ type: Optional[str] = Field( - default="Point", + default=None, title="type", frozen=True ) - coordinates: List[float] = Field( - default=None, - title="Geo property coordinates", - description="the actual coordinates" - ) - @field_validator("type") - @classmethod - def check_geoproperty_value_type(cls, value): - """ - Force property type to be "Point" - Args: - value: value field - Returns: - value - """ - if not value == "Point": - logging.warning(msg='NGSI_LD GeoProperty values must have type "Point"') - value = "Point" - return value + model_config = ConfigDict(extra='allow') - @field_validator("coordinates") - @classmethod - def check_geoproperty_value_coordinates(cls, value): + @model_validator(mode='after') + def check_geoproperty_value(self) -> Self: """ - Force property coordinates to be lis of two floats - Args: - value: value field - Returns: - value + Check if the value is a valid GeoProperty """ - if not isinstance(value, list) or len(value) != 2: - logging.error(msg='NGSI_LD GeoProperty values must have coordinates as list with length two') - raise ValueError - for element in value: - if not isinstance(element, float): - logging.error(msg='NGSI_LD GeoProperty values must have coordinates as list of floats') - raise TypeError - return value + if self.model_dump().get("type") == "Point": + return Point(**self.model_dump()) + elif self.model_dump().get("type") == "MultiPoint": + return MultiPoint(**self.model_dump()) + elif self.model_dump().get("type") == "LineString": + return LineString(**self.model_dump()) + elif self.model_dump().get("type") == "MultiLineString": + return MultiLineString(**self.model_dump()) + elif self.model_dump().get("type") == "Polygon": + return Polygon(**self.model_dump()) + elif self.model_dump().get("type") == "MultiPolygon": + return MultiPolygon(**self.model_dump()) + elif self.model_dump().get("type") == "GeometryCollection": + return GeometryCollection(**self.model_dump()) class ContextGeoProperty(BaseModel): @@ -263,15 +249,12 @@ def check_geoproperty_type(cls, value): value """ if not value == "GeoProperty": - if value == "Relationship": - value == "Relationship" - elif value == "TemporalProperty": - value == "TemporalProperty" - else: - logging.warning(msg='NGSI_LD GeoProperties must have type "GeoProperty" ' - '-> They are checked first, so if no GeoProperties are used ignore this warning!') - raise ValueError('NGSI_LD GeoProperties must have type "GeoProperty" ' - '-> They are checked first, so if no GeoProperties are used ignore this warning!') + logging.warning(msg='NGSI_LD GeoProperties must have type "GeoProperty" ' + '-> They are checked first, so if no GeoProperties are ' + 'used ignore this warning!') + raise ValueError('NGSI_LD GeoProperties must have type "GeoProperty" ' + '-> They are checked first, so if no GeoProperties are used' + ' ignore this warning!') return value @@ -691,8 +674,11 @@ def delete_properties(self, props: Union[Dict[str, ContextProperty], for name in names: delattr(self, name) - def add_properties(self, attrs: Union[Dict[str, ContextProperty], - List[NamedContextProperty]]) -> None: + def add_properties(self, attrs: Union[Dict[str, Union[ContextProperty, + ContextGeoProperty]], + List[Union[NamedContextProperty, + NamedContextGeoProperty]] + ]) -> None: """ Add property to entity Args: diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index 2dfd873a..0455bbde 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -4,10 +4,12 @@ import unittest +from geojson_pydantic import Point, MultiPoint, LineString, Polygon, GeometryCollection from pydantic import ValidationError from filip.models.ngsi_ld.context import \ - ContextLDEntity, ContextProperty, NamedContextProperty + ContextLDEntity, ContextProperty, NamedContextProperty, \ + ContextGeoPropertyValue, ContextGeoProperty class TestLDContextModels(unittest.TestCase): @@ -163,6 +165,79 @@ def setUp(self) -> None: ], } } + self.testpoint_value = { + "type": "Point", + "coordinates": [-8.5, 41.2] + } + self.testmultipoint_value = { + "type": "MultiPoint", + "coordinates": [ + [-3.80356167695194, 43.46296641666926], + [-3.804056, 43.464638] + ] + } + self.testlinestring_value = { + "type": "LineString", + "coordinates": [ + [-3.80356167695194, 43.46296641666926], + [-3.804056, 43.464638] + ] + } + self.testpolygon_value = { + "type": "Polygon", + "coordinates": [ + [ + [-3.80356167695194, 43.46296641666926], + [-3.804056, 43.464638], + [-3.805056, 43.463638], + [-3.80356167695194, 43.46296641666926] + ] + ] + } + self.testgeometrycollection_value = { + "type": "GeometryCollection", + "geometries": [ + { + "type": "Point", + "coordinates": [-3.80356167695194, 43.46296641666926] + }, + { + "type": "LineString", + "coordinates": [ + [-3.804056, 43.464638], + [-3.805056, 43.463638] + ] + } + ] + } + self.entity_geo_dict = { + "id": "urn:ngsi-ld:Geometry:001", + "type": "MyGeometry", + "testpoint": { + "type": "GeoProperty", + "value": self.testpoint_value + }, + "testmultipoint": { + "type": "GeoProperty", + "value": self.testmultipoint_value, + "observedAt": "2023-09-12T12:35:00Z" + }, + "testlinestring": { + "type": "GeoProperty", + "value": self.testlinestring_value, + "observedAt": "2023-09-12T12:35:30Z" + }, + "testpolygon": { + "type": "GeoProperty", + "value": self.testpolygon_value, + "observedAt": "2023-09-12T12:36:00Z" + }, + "testgeometrycollection": { + "type": "GeoProperty", + "value": self.testgeometrycollection_value, + "observedAt": "2023-09-12T12:36:30Z" + } + } def test_cb_attribute(self) -> None: """ @@ -181,6 +256,42 @@ def test_entity_id(self) -> None: with self.assertRaises(ValidationError): ContextLDEntity(**{'id': 'MyId', 'type': 'MyType'}) + def test_geo_property(self) -> None: + """ + Test ContextGeoPropertyValue models + Returns: + None + """ + geo_entity = ContextLDEntity(**self.entity_geo_dict) + new_entity = ContextLDEntity(id="urn:ngsi-ld:Geometry:002", type="MyGeometry") + test_point = NamedContextProperty( + name="testpoint", + type="GeoProperty", + value=Point(**self.testpoint_value) + ) + test_MultiPoint = NamedContextProperty( + name="testmultipoint", + type="GeoProperty", + value=MultiPoint(**self.testmultipoint_value) + ) + test_LineString = NamedContextProperty( + name="testlinestring", + type="GeoProperty", + value=LineString(**self.testlinestring_value) + ) + test_Polygon = NamedContextProperty( + name="testpolygon", + type="Polygon", + value=Polygon(**self.testpolygon_value) + ) + test_GeometryCollection = NamedContextProperty( + name="testgeometrycollection", + type="GeometryCollection", + value=GeometryCollection(**self.testgeometrycollection_value) + ) + new_entity.add_properties([test_point, test_MultiPoint, test_LineString, + test_Polygon, test_GeometryCollection]) + def test_cb_entity(self) -> None: """ Test context entity models From edae39ccedf8ee08e0a80a3a6fba4e1a5ccd46b6 Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Wed, 18 Sep 2024 17:44:03 +0200 Subject: [PATCH 10/16] feat: add function to add geo_properties --- filip/models/ngsi_ld/context.py | 38 +++++++++++++++++++--------- tests/models/test_ngsi_ld_context.py | 20 +++++++-------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 760bd13e..04270f73 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -121,8 +121,6 @@ def check_property_type(cls, value): return value - - class NamedContextProperty(ContextProperty): """ Context properties are properties of context entities. For example, the current speed of a car could be modeled @@ -175,14 +173,14 @@ def check_geoproperty_value(self) -> Self: """ if self.model_dump().get("type") == "Point": return Point(**self.model_dump()) - elif self.model_dump().get("type") == "MultiPoint": - return MultiPoint(**self.model_dump()) elif self.model_dump().get("type") == "LineString": return LineString(**self.model_dump()) - elif self.model_dump().get("type") == "MultiLineString": - return MultiLineString(**self.model_dump()) elif self.model_dump().get("type") == "Polygon": return Polygon(**self.model_dump()) + elif self.model_dump().get("type") == "MultiPoint": + return MultiPoint(**self.model_dump()) + elif self.model_dump().get("type") == "MultiLineString": + return MultiLineString(**self.model_dump()) elif self.model_dump().get("type") == "MultiPolygon": return MultiPolygon(**self.model_dump()) elif self.model_dump().get("type") == "GeometryCollection": @@ -215,7 +213,10 @@ class ContextGeoProperty(BaseModel): title="type", frozen=True ) - value: Optional[ContextGeoPropertyValue] = Field( + value: Optional[Union[ContextGeoPropertyValue, + Point, LineString, Polygon, + MultiPoint, MultiPolygon, + MultiLineString, GeometryCollection]] = Field( default=None, title="GeoProperty value", description="the actual data" @@ -674,11 +675,24 @@ def delete_properties(self, props: Union[Dict[str, ContextProperty], for name in names: delattr(self, name) - def add_properties(self, attrs: Union[Dict[str, Union[ContextProperty, - ContextGeoProperty]], - List[Union[NamedContextProperty, - NamedContextGeoProperty]] - ]) -> None: + def add_geo_properties(self, attrs: Union[Dict[str, ContextGeoProperty], + List[NamedContextGeoProperty]]) -> None: + """ + Add property to entity + Args: + attrs: + Returns: + None + """ + if isinstance(attrs, list): + attrs = {attr.name: ContextGeoProperty(**attr.model_dump(exclude={'name'}, + exclude_unset=True)) + for attr in attrs} + for key, attr in attrs.items(): + self.__setattr__(name=key, value=attr) + + def add_properties(self, attrs: Union[Dict[str, ContextProperty], + List[NamedContextProperty]]) -> None: """ Add property to entity Args: diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index 0455bbde..de14f1de 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -9,7 +9,7 @@ from filip.models.ngsi_ld.context import \ ContextLDEntity, ContextProperty, NamedContextProperty, \ - ContextGeoPropertyValue, ContextGeoProperty + ContextGeoPropertyValue, ContextGeoProperty, NamedContextGeoProperty class TestLDContextModels(unittest.TestCase): @@ -264,33 +264,33 @@ def test_geo_property(self) -> None: """ geo_entity = ContextLDEntity(**self.entity_geo_dict) new_entity = ContextLDEntity(id="urn:ngsi-ld:Geometry:002", type="MyGeometry") - test_point = NamedContextProperty( + test_point = NamedContextGeoProperty( name="testpoint", type="GeoProperty", value=Point(**self.testpoint_value) ) - test_MultiPoint = NamedContextProperty( + test_MultiPoint = NamedContextGeoProperty( name="testmultipoint", type="GeoProperty", value=MultiPoint(**self.testmultipoint_value) ) - test_LineString = NamedContextProperty( + test_LineString = NamedContextGeoProperty( name="testlinestring", type="GeoProperty", value=LineString(**self.testlinestring_value) ) - test_Polygon = NamedContextProperty( + test_Polygon = NamedContextGeoProperty( name="testpolygon", - type="Polygon", + type="GeoProperty", value=Polygon(**self.testpolygon_value) ) - test_GeometryCollection = NamedContextProperty( + test_GeometryCollection = NamedContextGeoProperty( name="testgeometrycollection", - type="GeometryCollection", + type="GeoProperty", value=GeometryCollection(**self.testgeometrycollection_value) ) - new_entity.add_properties([test_point, test_MultiPoint, test_LineString, - test_Polygon, test_GeometryCollection]) + new_entity.add_geo_properties([test_point, test_MultiPoint, test_LineString, + test_Polygon, test_GeometryCollection]) def test_cb_entity(self) -> None: """ From 33bef2e489a8a15aca6e8763980599d372d745c1 Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Wed, 2 Oct 2024 18:27:26 +0200 Subject: [PATCH 11/16] fix: validation of entity attributes --- filip/models/ngsi_ld/context.py | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 04270f73..cbd77960 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -239,25 +239,6 @@ class ContextGeoProperty(BaseModel): ) field_validator("datasetId")(validate_fiware_datatype_string_protect) - @field_validator("type") - @classmethod - def check_geoproperty_type(cls, value): - """ - Force property type to be "GeoProperty" - Args: - value: value field - Returns: - value - """ - if not value == "GeoProperty": - logging.warning(msg='NGSI_LD GeoProperties must have type "GeoProperty" ' - '-> They are checked first, so if no GeoProperties are ' - 'used ignore this warning!') - raise ValueError('NGSI_LD GeoProperties must have type "GeoProperty" ' - '-> They are checked first, so if no GeoProperties are used' - ' ignore this warning!') - return value - class NamedContextGeoProperty(ContextGeoProperty): """ @@ -522,7 +503,6 @@ def __init__(self, data.update(self._validate_attributes(data)) super().__init__(id=id, type=type, **data) - # TODO we should distinguish between context relationship @classmethod def _validate_attributes(cls, data: Dict): fields = set([field.validation_alias for (_, field) in cls.model_fields.items()] + @@ -535,10 +515,15 @@ def _validate_attributes(cls, data: Dict): for key, attr in data.items(): # Check if the keyword is not already present in the fields if key not in fields: - try: + if attr.get("type") == "Relationship": + attrs[key] = ContextRelationship.model_validate(attr) + elif attr.get("type") == "GeoProperty": attrs[key] = ContextGeoProperty.model_validate(attr) - except ValueError: + elif attr.get("type") == "Property": attrs[key] = ContextProperty.model_validate(attr) + else: + raise ValueError(f"Attribute {attr.get('type')} " + "is not a valid type") return attrs model_config = ConfigDict(extra='allow', validate_default=True, validate_assignment=True) From 0e72383895123635a8eee5926b9e74a6b5cc1fb7 Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Wed, 2 Oct 2024 18:28:57 +0200 Subject: [PATCH 12/16] fix: change coordinates from list to tuple --- tests/models/test_ngsi_ld_context.py | 49 ++++++++++++++-------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index de14f1de..535d8511 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -1,7 +1,6 @@ """ Test module for context broker models """ - import unittest from geojson_pydantic import Point, MultiPoint, LineString, Polygon, GeometryCollection @@ -50,7 +49,7 @@ def setUp(self) -> None: "type": "GeoProperty", "value": { "type": "Point", - "coordinates": [-8.5, 41.2] + "coordinates": (-8.5, 41.2) # coordinates are normally a tuple } }, "@context": [ @@ -63,7 +62,7 @@ def setUp(self) -> None: "type": "GeoProperty", "value": { "type": "Point", - "coordinates": [-8.5, 41.2] + "coordinates": (-8.5, 41.2) } }, "totalSpotNumber": { @@ -167,46 +166,46 @@ def setUp(self) -> None: } self.testpoint_value = { "type": "Point", - "coordinates": [-8.5, 41.2] + "coordinates": (-8.5, 41.2) } self.testmultipoint_value = { "type": "MultiPoint", - "coordinates": [ - [-3.80356167695194, 43.46296641666926], - [-3.804056, 43.464638] - ] + "coordinates": ( + (-3.80356167695194, 43.46296641666926), + (-3.804056, 43.464638) + ) } self.testlinestring_value = { "type": "LineString", - "coordinates": [ - [-3.80356167695194, 43.46296641666926], - [-3.804056, 43.464638] - ] + "coordinates": ( + (-3.80356167695194, 43.46296641666926), + (-3.804056, 43.464638) + ) } self.testpolygon_value = { "type": "Polygon", - "coordinates": [ - [ - [-3.80356167695194, 43.46296641666926], - [-3.804056, 43.464638], - [-3.805056, 43.463638], - [-3.80356167695194, 43.46296641666926] - ] - ] + "coordinates": ( + ( + (-3.80356167695194, 43.46296641666926), + (-3.804056, 43.464638), + (-3.805056, 43.463638), + (-3.80356167695194, 43.46296641666926) + ) + ) } self.testgeometrycollection_value = { "type": "GeometryCollection", "geometries": [ { "type": "Point", - "coordinates": [-3.80356167695194, 43.46296641666926] + "coordinates": (-3.80356167695194, 43.46296641666926) }, { "type": "LineString", - "coordinates": [ - [-3.804056, 43.464638], - [-3.805056, 43.463638] - ] + "coordinates": ( + (-3.804056, 43.464638), + (-3.805056, 43.463638) + ) } ] } From 9e8beffd4a6f49198597ca33e93a6cee438ec46f Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Tue, 8 Oct 2024 16:35:27 +0200 Subject: [PATCH 13/16] fix: categorizing nested property --- filip/models/ngsi_ld/context.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index 2992390d..fdfb17a0 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -502,32 +502,41 @@ def __init__(self, # TODO should geoproperty has subproperties? and can geoproperty be subproperties? @classmethod - def _validate_single_property(cls, key, attr): - subattrs = {} + def _validate_single_property(cls, attr) -> ContextProperty: + property_fields = set([field.validation_alias + for (_, field) in ContextProperty.model_fields.items()] + + [field_name for field_name in ContextProperty.model_fields]) + property_fields.remove(None) + # subattrs = {} if attr.get("type") == "Relationship": - attr = ContextRelationship.model_validate(attr) + attr_instance = ContextRelationship.model_validate(attr) elif attr.get("type") == "GeoProperty": - attrs[key] = ContextGeoProperty.model_validate(attr) + attr_instance = ContextGeoProperty.model_validate(attr) elif attr.get("type") == "Property": - attrs[key] = ContextProperty.model_validate(attr) + attr_instance = ContextProperty.model_validate(attr) else: raise ValueError(f"Attribute {attr.get('type')} " "is not a valid type") - return subattrs + for subkey, subattr in attr.items(): + # TODO can we ensure that the subattr can only be dict? + if isinstance(subattr, dict) and subkey not in property_fields: + attr_instance.model_extra.update( + {subkey: cls._validate_single_property(attr=subattr)} + ) + return attr_instance - # TODO is "validate_attributes" still relevant for LD entities? @classmethod def _validate_attributes(cls, data: Dict): - fields = set([field.validation_alias for (_, field) in cls.model_fields.items()] + + entity_fields = set([field.validation_alias for (_, field) in cls.model_fields.items()] + [field_name for field_name in cls.model_fields]) - fields.remove(None) + entity_fields.remove(None) # Initialize the attribute dictionary attrs = {} # Iterate through the data for key, attr in data.items(): # Check if the keyword is not already present in the fields - if key not in fields: # TODO why ignoring all in fields? - attrs[key] = cls._validate_single_property(key, attr) + if key not in entity_fields: + attrs[key] = cls._validate_single_property(attr=attr) return attrs model_config = ConfigDict(extra='allow', validate_default=True, validate_assignment=True) From 42a0de60b884d5057bdf7b8b58cea08a0cb2eaf9 Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Tue, 22 Oct 2024 15:30:14 +0200 Subject: [PATCH 14/16] fix: get context properties --- filip/models/ngsi_ld/context.py | 85 +++++++++++++++------------- tests/models/test_ngsi_ld_context.py | 28 ++++----- 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index fdfb17a0..fa923cc8 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -19,7 +19,7 @@ class DataTypeLD(str, Enum): In NGSI-LD the data types on context entities are only divided into properties and relationships. """ _init_ = 'value __doc__' - + GEOPROPERTY = "GeoProperty", "A property that represents a geometry value" PROPERTY = "Property", "All attributes that do not represent a relationship" RELATIONSHIP = "Relationship", "Reference to another context entity, which can be identified with a URN." @@ -99,6 +99,15 @@ class ContextProperty(BaseModel): ) field_validator("datasetId")(validate_fiware_datatype_string_protect) + @classmethod + def get_model_fields_set(cls): + """ + Get all names and aliases of the model fields. + """ + return set([field.validation_alias + for (_, field) in cls.model_fields.items()] + + [field_name for field_name in cls.model_fields]) + @field_validator("type") @classmethod def check_property_type(cls, value): @@ -192,7 +201,7 @@ class ContextGeoProperty(BaseModel): Example: - "location": { + { "type": "GeoProperty", "value": { "type": "Point", @@ -201,7 +210,6 @@ class ContextGeoProperty(BaseModel): 43.46296641666926 ] } - } """ model_config = ConfigDict(extra='allow') @@ -500,18 +508,28 @@ def __init__(self, data.update(self._validate_attributes(data)) super().__init__(id=id, type=type, **data) + @classmethod + def get_model_fields_set(cls): + """ + Get all names and aliases of the model fields. + """ + return set([field.validation_alias + for (_, field) in cls.model_fields.items()] + + [field_name for field_name in cls.model_fields]) + # TODO should geoproperty has subproperties? and can geoproperty be subproperties? @classmethod def _validate_single_property(cls, attr) -> ContextProperty: - property_fields = set([field.validation_alias - for (_, field) in ContextProperty.model_fields.items()] + - [field_name for field_name in ContextProperty.model_fields]) + property_fields = ContextProperty.get_model_fields_set() property_fields.remove(None) # subattrs = {} if attr.get("type") == "Relationship": attr_instance = ContextRelationship.model_validate(attr) elif attr.get("type") == "GeoProperty": - attr_instance = ContextGeoProperty.model_validate(attr) + try: + attr_instance = ContextGeoProperty.model_validate(attr) + except Exception as e: + pass elif attr.get("type") == "Property": attr_instance = ContextProperty.model_validate(attr) else: @@ -527,8 +545,7 @@ def _validate_single_property(cls, attr) -> ContextProperty: @classmethod def _validate_attributes(cls, data: Dict): - entity_fields = set([field.validation_alias for (_, field) in cls.model_fields.items()] + - [field_name for field_name in cls.model_fields]) + entity_fields = cls.get_model_fields_set() entity_fields.remove(None) # Initialize the attribute dictionary attrs = {} @@ -574,30 +591,26 @@ def get_properties(self, if response_format == PropertyFormat.DICT: final_dict = {} for key, value in self.model_dump(exclude_unset=True).items(): - if key not in ContextLDEntity.model_fields: - try: - if value.get('type') != DataTypeLD.RELATIONSHIP: - try: - final_dict[key] = ContextGeoProperty(**value) - except ValueError: # if context attribute - final_dict[key] = ContextProperty(**value) - except AttributeError: - if isinstance(value, list): - pass + if key not in ContextLDEntity.get_model_fields_set(): + if value.get('type') != DataTypeLD.RELATIONSHIP: + if value.get('type') == DataTypeLD.GEOPROPERTY: + final_dict[key] = ContextGeoProperty(**value) + elif value.get('type') == DataTypeLD.PROPERTY: + final_dict[key] = ContextProperty(**value) + else: # named context property by default + final_dict[key] = ContextProperty(**value) return final_dict # response format list: final_list = [] for key, value in self.model_dump(exclude_unset=True).items(): - if key not in ContextLDEntity.model_fields: - try: - if value.get('type') != DataTypeLD.RELATIONSHIP: - try: - final_list.append(NamedContextGeoProperty(name=key, **value)) - except ValueError: # if context attribute - final_list.append(NamedContextProperty(name=key, **value)) - except AttributeError: - if isinstance(value, list): - pass + if key not in ContextLDEntity.get_model_fields_set(): + if value.get('type') != DataTypeLD.RELATIONSHIP: + if value.get('type') == DataTypeLD.GEOPROPERTY: + final_list.append(NamedContextGeoProperty(name=key, **value)) + elif value.get('type') == DataTypeLD.PROPERTY: + final_list.append(NamedContextProperty(name=key, **value)) + else: # named context property by default + final_list.append(NamedContextProperty(name=key, **value)) return final_list def add_attributes(self, **kwargs): @@ -728,7 +741,7 @@ def get_relationships(self, if response_format == PropertyFormat.DICT: final_dict = {} for key, value in self.model_dump(exclude_unset=True).items(): - if key not in ContextLDEntity.model_fields: + if key not in ContextLDEntity.get_model_fields_set(): try: if value.get('type') == DataTypeLD.RELATIONSHIP: final_dict[key] = ContextRelationship(**value) @@ -739,13 +752,9 @@ def get_relationships(self, # response format list: final_list = [] for key, value in self.model_dump(exclude_unset=True).items(): - if key not in ContextLDEntity.model_fields: - try: - if value.get('type') == DataTypeLD.RELATIONSHIP: - final_list.append(NamedContextRelationship(name=key, **value)) - except AttributeError: # if context attribute - if isinstance(value, list): - pass + if key not in ContextLDEntity.get_model_fields_set(): + if value.get('type') == DataTypeLD.RELATIONSHIP: + final_list.append(NamedContextRelationship(name=key, **value)) return final_list def get_context(self): @@ -758,7 +767,7 @@ def get_context(self): """ found_list = False for key, value in self.model_dump(exclude_unset=True).items(): - if key not in ContextLDEntity.model_fields: + if key not in ContextLDEntity.get_model_fields_set(): if isinstance(value, list): found_list = True return value diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index 7cf78cf9..31a2eca8 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -190,14 +190,14 @@ def setUp(self) -> None: } self.testpolygon_value = { "type": "Polygon", - "coordinates": ( - ( + "coordinates": [ + [ (-3.80356167695194, 43.46296641666926), (-3.804056, 43.464638), (-3.805056, 43.463638), (-3.80356167695194, 43.46296641666926) - ) - ) + ] + ] } self.testgeometrycollection_value = { "type": "GeometryCollection", @@ -257,10 +257,6 @@ def test_cb_property(self) -> None: prop = ContextProperty(**{'value': 20}) self.assertIsInstance(prop.value, int) - def test_entity_id(self) -> None: - with self.assertRaises(ValidationError): - ContextLDEntity(**{'id': 'MyId', 'type': 'MyType'}) - def test_geo_property(self) -> None: """ Test ContextGeoPropertyValue models @@ -303,6 +299,7 @@ def test_cb_entity(self) -> None: Returns: None """ + test = ContextLDEntity.get_model_fields_set() entity1 = ContextLDEntity(**self.entity1_dict) entity2 = ContextLDEntity(**self.entity2_dict) @@ -363,14 +360,19 @@ def test_get_properties(self): """ Test the get_properties method """ - pass - entity = ContextLDEntity(id="urn:ngsi-ld:test", type="Tester") + entity = ContextLDEntity(id="urn:ngsi-ld:test", + type="Tester", + hasLocation={ + "type": "Relationship", + "object": "urn:ngsi-ld:test2" + }) properties = [ NamedContextProperty(name="prop1"), NamedContextProperty(name="prop2"), ] entity.add_properties(properties) + entity.get_properties(response_format="list") self.assertEqual(entity.get_properties(response_format="list"), properties) @@ -378,11 +380,11 @@ def test_entity_delete_properties(self): """ Test the delete_properties method """ - prop = ContextProperty(**{'value': 20, 'type': 'Text'}) + prop = ContextProperty(**{'value': 20, 'type': 'Property'}) named_prop = NamedContextProperty(**{'name': 'test2', 'value': 20, - 'type': 'Text'}) - prop3 = ContextProperty(**{'value': 20, 'type': 'Text'}) + 'type': 'Property'}) + prop3 = ContextProperty(**{'value': 20, 'type': 'Property'}) entity = ContextLDEntity(id="urn:ngsi-ld:12", type="Test") From 79937c4e97f4597de123a6c473f36adab8c171dd Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Tue, 22 Oct 2024 16:25:16 +0200 Subject: [PATCH 15/16] fix: get context --- filip/models/ngsi_ld/context.py | 13 ++--- tests/models/test_ngsi_ld_context.py | 77 +++++++++++++--------------- 2 files changed, 42 insertions(+), 48 deletions(-) diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py index fa923cc8..b2ce89a4 100644 --- a/filip/models/ngsi_ld/context.py +++ b/filip/models/ngsi_ld/context.py @@ -765,15 +765,12 @@ def get_context(self): Returns: context of the entity as list """ - found_list = False - for key, value in self.model_dump(exclude_unset=True).items(): - if key not in ContextLDEntity.get_model_fields_set(): - if isinstance(value, list): - found_list = True - return value - if not found_list: + _, context = self.model_dump(include={"context"}).popitem() + if not context: logging.warning("No context in entity") - return None + return None + else: + return context class ActionTypeLD(str, Enum): diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py index 31a2eca8..7a26dbb3 100644 --- a/tests/models/test_ngsi_ld_context.py +++ b/tests/models/test_ngsi_ld_context.py @@ -133,43 +133,30 @@ def setUp(self) -> None: } } } - # The entity for testing the nested structure of properties - self.entity_sub_props_dict_wrong = { - "id": "urn:ngsi-ld:OffStreetParking:Downtown1", - "type": "OffStreetParking", - "name": { - "type": "Property", - "value": "Downtown One" - }, - "availableSpotNumber": { - "type": "Property", - "value": 121, - "observedAt": "2017-07-29T12:05:02Z", - "reliability": { - "type": "NotAProperty", - "value": 0.7 - }, - "providedBy": { - "type": "NotARelationship", - "object": "urn:ngsi-ld:Camera:C1" - } - }, - "totalSpotNumber": { - "type": "Property", - "value": 200 - }, - "location": { - "type": "GeoProperty", - "value": { - "type": "Point", - "coordinates": [-8.5, 41.2] - } - }, - "@context": [ - "http://example.org/ngsi-ld/latest/parking.jsonld", - "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" - ] - } + # # The entity for testing the nested structure of properties + # self.entity_sub_props_dict_wrong = { + # "id": "urn:ngsi-ld:OffStreetParking:Downtown1", + # "type": "OffStreetParking", + # "name": { + # "type": "Property", + # "value": "Downtown One" + # }, + # "totalSpotNumber": { + # "type": "Property", + # "value": 200 + # }, + # "location": { + # "type": "GeoProperty", + # "value": { + # "type": "Point", + # "coordinates": [-8.5, 41.2] + # } + # }, + # "@context": [ + # "http://example.org/ngsi-ld/latest/parking.jsonld", + # "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + # ] + # } self.testpoint_value = { "type": "Point", "coordinates": (-8.5, 41.2) @@ -354,7 +341,16 @@ def test_validate_subproperties_dict_wrong(self) -> None: Returns: None """ - entity5 = ContextLDEntity(**self.entity_sub_props_dict_wrong) + entity_sub_props_dict_wrong_1 = self.entity1_dict.copy() + entity_sub_props_dict_wrong_1[ + "availableSpotNumber"]["reliability"]["type"] = "NotProperty" + with self.assertRaises(ValueError): + entity5 = ContextLDEntity(**entity_sub_props_dict_wrong_1) + entity_sub_props_dict_wrong_2 = self.entity1_dict.copy() + entity_sub_props_dict_wrong_2[ + "availableSpotNumber"]["providedBy"]["type"] = "NotRelationship" + with self.assertRaises(ValueError): + entity5 = ContextLDEntity(**entity_sub_props_dict_wrong_2) def test_get_properties(self): """ @@ -414,9 +410,10 @@ def test_get_context(self): self.assertEqual(self.entity1_context, context_entity1) - # test here if entity without context can be validated and get_context works accordingly: + # test here if entity without context can be validated and get_context + # works accordingly: entity3 = ContextLDEntity(**self.entity3_dict) context_entity3 = entity3.get_context() self.assertEqual(None, - context_entity3) \ No newline at end of file + context_entity3) From 8949dbe5e9a3f1705ffcaaad01afe34bd4ad3acb Mon Sep 17 00:00:00 2001 From: JunsongDu Date: Tue, 22 Oct 2024 16:29:17 +0200 Subject: [PATCH 16/16] fix: remove unnecessary clean up func from tests --- tests/models/test_ngsi_ld_subscriptions.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/models/test_ngsi_ld_subscriptions.py b/tests/models/test_ngsi_ld_subscriptions.py index e02f8ffc..aa4d76a0 100644 --- a/tests/models/test_ngsi_ld_subscriptions.py +++ b/tests/models/test_ngsi_ld_subscriptions.py @@ -231,11 +231,4 @@ def test_query_string_serialization(self): examples[9] = 'address[city]=="Berlin".' examples[10] = 'sensor.rawdata[airquality.particulate]==40' for example in examples.values(): - validate_ngsi_ld_query(example) - - def tearDown(self) -> None: - """ - Cleanup test server - """ - clear_all(fiware_header=self.fiware_header, - cb_url=settings.CB_URL) \ No newline at end of file + validate_ngsi_ld_query(example) \ No newline at end of file