From 6231093ffe89f9e45e6ad345ee4bbfd86b781371 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 4 Aug 2024 13:35:29 -0400 Subject: [PATCH 1/5] minor STAC API updates --- pycsw/ogc/api/records.py | 10 +++++++--- pycsw/ogc/api/util.py | 5 ++++- pycsw/stac/api.py | 12 ++++++++++-- .../suites/stac_api/test_stac_api_functional.py | 13 +++++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/pycsw/ogc/api/records.py b/pycsw/ogc/api/records.py index 4533c9d25..be3b76957 100644 --- a/pycsw/ogc/api/records.py +++ b/pycsw/ogc/api/records.py @@ -1328,11 +1328,15 @@ def record2json(record, url, collection, mode='ogcapi-records'): } if mode == 'stac-api': - record_dict['properties']['datetime'] = to_rfc3339(record.date) + date_, date_type = to_rfc3339(record.date) + record_dict['properties']['datetime'] = date_ if None not in [record.time_begin, record.time_end]: - record_dict['properties']['start_datetime'] = to_rfc3339(record.time_begin) - record_dict['properties']['end_datetime'] = to_rfc3339(record.time_end) + start_date, start_date_type = to_rfc3339(record.time_begin) + end_date, end_date_type = to_rfc3339(record.time_end) + + record_dict['properties']['start_datetime'] = start_date + record_dict['properties']['end_datetime'] = end_date return record_dict diff --git a/pycsw/ogc/api/util.py b/pycsw/ogc/api/util.py index 1d91771c5..63f6cfec5 100644 --- a/pycsw/ogc/api/util.py +++ b/pycsw/ogc/api/util.py @@ -91,7 +91,10 @@ def json_serial(obj): """ if isinstance(obj, (datetime, date, time)): - return obj.isoformat() + 'Z' + if isinstance(obj, date): + return obj.strftime('%Y-%m-%d') + else: + return obj.isoformat() + 'Z' elif isinstance(obj, bytes): try: LOGGER.debug('Returning as UTF-8 decoded bytes') diff --git a/pycsw/stac/api.py b/pycsw/stac/api.py index d50979dbc..5a9c6c8f5 100644 --- a/pycsw/stac/api.py +++ b/pycsw/stac/api.py @@ -264,6 +264,9 @@ def collections(self, headers_, args): 'href': self.config['server']['url'] }] + response['numberMatched'] = len(response['collections']) + response['numberReturned'] = len(response['collections']) + return self.get_response(200, headers_, response) def collection(self, headers_, args, collection='metadata:main'): @@ -345,7 +348,7 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'): headers, status, response = super().items(headers_, json_post_data, args, collection) if collection not in self.get_all_collections(): - msg = f'Invalid collection' + msg = 'Invalid collection' LOGGER.exception(msg) return self.get_exception(400, headers_, 'InvalidParameterValue', msg) @@ -373,6 +376,11 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'): else: response2['features'].append(record) + for link in record['links']: + if link.get('rel') is None: + LOGGER.debug('Missing link relation; adding rel=related') + link['rel'] = 'related' + for count, value in enumerate(response2['links']): if value['rel'] == 'alternate': response2['links'].pop(count) @@ -402,7 +410,7 @@ def item(self, headers_, args, collection, item): headers, status, response = super().item(headers_, args, collection, item) if collection not in self.get_all_collections(): - msg = f'Invalid collection' + msg = 'Invalid collection' LOGGER.exception(msg) return self.get_exception(400, headers_, 'InvalidParameterValue', msg) diff --git a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py index 385e65080..5228310d3 100644 --- a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py +++ b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py @@ -83,6 +83,19 @@ def test_conformance(config): assert conformance in content['conformsTo'] +def test_collections(config): + api = STACAPI(config) + headers, status, content = api.collections({}, {'f': 'json'}) + content = json.loads(content) + + assert headers['Content-Type'] == 'application/json' + assert status == 200 + assert len(content['links']) == 3 + + assert len(content['collections']) == 1 + assert len(content['collections']) == content['numberMatched'] + assert len(content['collections']) == content['numberReturned'] + def test_queryables(config): api = STACAPI(config) headers, status, content = api.queryables({}, {}) From b14a7656d2ea26afeefb273104936498738e5c8b Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 4 Aug 2024 13:41:48 -0400 Subject: [PATCH 2/5] add tests for required link properties --- .../suites/stac_api/test_stac_api_functional.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py index 5228310d3..9e4bc44b4 100644 --- a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py +++ b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py @@ -131,6 +131,11 @@ def test_items(config): assert record['stac_version'] == '1.0.0' assert record['collection'] == 'metadata:main' + for feature in content['features']: + for link in feature['links']: + assert 'href' in link + assert 'rel' in link + # test GET KVP requests content = json.loads(api.items({}, None, {'bbox': '-180,-90,180,90'})[2]) assert len(content['features']) == 3 @@ -207,5 +212,9 @@ def test_item(config): assert content['stac_version'] == '1.0.0' assert content['collection'] == 'metadata:main' + for link in content['links']: + assert 'href' in link + assert 'rel' in link + headers, status, content = api.item({}, {}, 'foo', item) assert status == 400 From 85364cb9ab64f11d29dec3c55b00e60f1bbd2963 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 4 Aug 2024 14:41:01 -0400 Subject: [PATCH 3/5] STAC API: add bbox is geometry is null (#847) --- pycsw/stac/api.py | 8 ++++++++ .../suites/stac_api/test_stac_api_functional.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/pycsw/stac/api.py b/pycsw/stac/api.py index 5a9c6c8f5..377a62b2b 100644 --- a/pycsw/stac/api.py +++ b/pycsw/stac/api.py @@ -38,6 +38,7 @@ from pycsw.ogc.api.oapi import gen_oapi from pycsw.ogc.api.records import API from pycsw.core.pygeofilter_evaluate import to_filter +from pycsw.core.util import geojson_geometry2bbox LOGGER = logging.getLogger(__name__) @@ -376,6 +377,13 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'): else: response2['features'].append(record) + if record.get('bbox') is None: + geometry = record.get('geometry') + if geometry is not None: + LOGGER.debug('Calculating bbox from geometry') + bbox = geojson_geometry2bbox(geometry) + record['bbox'] = [float(t) for t in bbox.split(',')] + for link in record['links']: if link.get('rel') is None: LOGGER.debug('Missing link relation; adding rel=related') diff --git a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py index 9e4bc44b4..ca340c319 100644 --- a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py +++ b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py @@ -132,6 +132,9 @@ def test_items(config): assert record['collection'] == 'metadata:main' for feature in content['features']: + if feature.get('geometry') is not None: + assert 'bbox' in feature + for link in feature['links']: assert 'href' in link assert 'rel' in link From 232123b7ca5592fb7b2e287a4882e7fbab594947 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 4 Aug 2024 14:53:28 -0400 Subject: [PATCH 4/5] safeguard null geometry handling on STAC parsing (#847) --- pycsw/core/metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pycsw/core/metadata.py b/pycsw/core/metadata.py index d12148736..e6fed516d 100644 --- a/pycsw/core/metadata.py +++ b/pycsw/core/metadata.py @@ -1769,7 +1769,8 @@ def _parse_oarec_record(context, repos, record): if links: _set(context, recobj, 'pycsw:Links', json.dumps(links)) - _set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(util.geojson_geometry2bbox(record['geometry']))) + if record['geometry'] is not None: + _set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(util.geojson_geometry2bbox(record['geometry']))) if 'temporal' in record['properties'].get('extent', []): _set(context, recobj, 'pycsw:TempExtent_begin', record['properties']['extent']['temporal']['interval'][0]) @@ -1784,6 +1785,7 @@ def _parse_stac_resource(context, repos, record): recobj = repos.dataset() keywords = [] links = [] + bbox_wkt = None stac_type = record.get('type', 'Feature') if stac_type == 'Feature': @@ -1793,7 +1795,8 @@ def _parse_stac_resource(context, repos, record): stype = 'item' title = record['properties'].get('title') abstract = record['properties'].get('description') - bbox_wkt = util.bbox2wktpolygon(util.geojson_geometry2bbox(record['geometry'])) + if record.get('geometry') is not None: + bbox_wkt = util.bbox2wktpolygon(util.geojson_geometry2bbox(record['geometry'])) elif stac_type == 'Collection': LOGGER.debug('Parsing STAC Collection') conformance = 'https://github.com/radiantearth/stac-spec/tree/master/collection-spec/collection-spec.md' @@ -1804,8 +1807,6 @@ def _parse_stac_resource(context, repos, record): if 'extent' in record and 'spatial' in record['extent']: bbox_csv = ','.join(str(t) for t in record['extent']['spatial']['bbox'][0]) bbox_wkt = util.bbox2wktpolygon(bbox_csv) - else: - bbox_wkt = None if 'extent' in record and 'temporal' in record['extent'] and 'interval' in record['extent']['temporal']: _set(context, recobj, 'pycsw:TempExtent_begin', record['extent']['temporal']['interval'][0][0]) _set(context, recobj, 'pycsw:TempExtent_end', record['extent']['temporal']['interval'][0][1]) @@ -1816,7 +1817,6 @@ def _parse_stac_resource(context, repos, record): stype = 'catalog' title = record.get('title') abstract = record.get('description') - bbox_wkt = None _set(context, recobj, 'pycsw:Identifier', record['id']) _set(context, recobj, 'pycsw:Typename', typename) From fca0fe96548cc8b4aac6026b6f9507d56deb9d73 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 4 Aug 2024 15:14:38 -0400 Subject: [PATCH 5/5] update geometry/bbox handling --- pycsw/core/metadata.py | 2 +- .../functionaltests/suites/stac_api/test_stac_api_functional.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pycsw/core/metadata.py b/pycsw/core/metadata.py index e6fed516d..7860c13fd 100644 --- a/pycsw/core/metadata.py +++ b/pycsw/core/metadata.py @@ -1769,7 +1769,7 @@ def _parse_oarec_record(context, repos, record): if links: _set(context, recobj, 'pycsw:Links', json.dumps(links)) - if record['geometry'] is not None: + if record.get('geometry') is not None: _set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(util.geojson_geometry2bbox(record['geometry']))) if 'temporal' in record['properties'].get('extent', []): diff --git a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py index ca340c319..102c729fb 100644 --- a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py +++ b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py @@ -134,6 +134,7 @@ def test_items(config): for feature in content['features']: if feature.get('geometry') is not None: assert 'bbox' in feature + assert isinstance(feature['bbox'], list) for link in feature['links']: assert 'href' in link