From 875ea3e2c6d94509b619a1d9785b11b0e5a7b69d Mon Sep 17 00:00:00 2001 From: froded Date: Fri, 10 Feb 2023 09:27:45 +0100 Subject: [PATCH 1/4] Initial release --- sardata/sardataNBS.py | 159 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 sardata/sardataNBS.py diff --git a/sardata/sardataNBS.py b/sardata/sardataNBS.py new file mode 100644 index 0000000..6059652 --- /dev/null +++ b/sardata/sardataNBS.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Feb 9 09:32:05 2023 + +@author: froded +""" + +from owslib import fes +from owslib.fes import SortBy, SortProperty +from owslib.csw import CatalogueServiceWeb + + +class SARData(): + """ + A class for getting Sentinel-1 netCDF data from the Norwegian Ground Segment (NBS) + + Parameters + ----------- + endpoint : str + URL to NBS. Default: https://nbs.csw.met.no/csw + + bbox : int list + A boudary box for search area speified in latitude and longitude + bbox = [lon_min, lat_min, lon_max, lat_max] + + Example: + >>> bbox = [-10, 75, 40, 85] + + start : datetime object + Specify the start time of the range to serchthe + + stop : datetime object + Specify the stop time of the range to serchthe + + Example: + >>> from datetime import datetime, timedelta + >>> stop = datetime(2010, 1, 1, 12, 30, 59).replace(tzinfo=pytz.utc) + >>> start = stop - timedelta(days=7) + + kw_names : str + A search string filter to limit the result of a search. + Example: + >>> kw_name='S1A*' + + """ + + def __init__(self, endpoint='https://nbs.csw.met.no/csw', bbox=None, start=None, stop=None, kw_names=None, crs="urn:ogc:def:crs:OGC:1.3:CRS84", *args, **kwargs): + + constraints = [] + csw = None + while csw is None: + try: + # connect + csw = CatalogueServiceWeb(endpoint, timeout=60) + except: + pass + + if kw_names: + kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText") + or_filt = fes.Or([fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in kw_names]) + constraints.append(or_filt) + + + + if all(v is not None for v in [start, stop]): + begin, end = self._fes_date_filter(start, stop) + constraints.append(begin) + constraints.append(end) + + if bbox: + bbox_crs = fes.BBox(bbox, crs=crs) + constraints.append(bbox_crs) + if len(constraints) >= 2: + filter_list = [ + fes.And( + constraints + ) + ] + else: + filter_list = constraints + + self._get_csw_records(csw, filter_list, pagesize=2, maxrecords=10) + self.csw = csw + url_opendap = [] + + for key, value in list(csw.records.items()): + for ref in value.references: + if ref['scheme'] == 'OPENDAP:OPENDAP': + url_opendap.append(ref['url']) + self.url_opendap = url_opendap + + + + + def _get_csw_records(self, csw, filter_list, pagesize=2, maxrecords=10): + """Iterate `maxrecords`/`pagesize` times until the requested value in + `maxrecords` is reached. + """ + # Iterate over sorted results. + sortby = SortBy([SortProperty("dc:title", "ASC")]) + csw_records = {} + startposition = 0 + nextrecord = getattr(csw, "results", 1) + while nextrecord != 0: + csw.getrecords2( + constraints=filter_list, + startposition=startposition, + maxrecords=pagesize, + sortby=sortby, + ) + csw_records.update(csw.records) + if csw.results["nextrecord"] == 0: + break + startposition += pagesize + 1 # Last one is included. + if startposition >= maxrecords: + break + csw.records.update(csw_records) + + + + + def _fes_date_filter(self, start, stop, constraint="within"): + """ + Take datetime-like objects and returns a fes filter for date range + (begin and end inclusive). + NOTE: Truncates the minutes!!! + """ + + start = start.strftime("%Y-%m-%d %H:00") + stop = stop.strftime("%Y-%m-%d %H:00") + if constraint == "overlaps": + propertyname = "apiso:TempExtent_begin" + begin = fes.PropertyIsLessThanOrEqualTo(propertyname=propertyname, literal=stop) + propertyname = "apiso:TempExtent_end" + end = fes.PropertyIsGreaterThanOrEqualTo( + propertyname=propertyname, literal=start + ) + elif constraint == "within": + propertyname = "apiso:TempExtent_begin" + begin = fes.PropertyIsGreaterThanOrEqualTo( + propertyname=propertyname, literal=start + ) + propertyname = "apiso:TempExtent_end" + end = fes.PropertyIsLessThanOrEqualTo(propertyname=propertyname, literal=stop) + else: + raise NameError("Unrecognized constraint {}".format(constraint)) + return begin, end + + + + + + + + + + + \ No newline at end of file From 4472e89637830bd2cd55cb704aac3a290540c655 Mon Sep 17 00:00:00 2001 From: froded Date: Fri, 10 Feb 2023 11:08:33 +0100 Subject: [PATCH 2/4] Added test for testing SARData --- tests/test_sarwind.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_sarwind.py b/tests/test_sarwind.py index 3cea935..be0f6c1 100644 --- a/tests/test_sarwind.py +++ b/tests/test_sarwind.py @@ -5,6 +5,9 @@ from sarwind.sarwind import SARWind +from sardata.sardataNBS import SARData +from datetime import datetime, timedelta +import pytz @pytest.mark.unittests @pytest.mark.sarwind @@ -79,3 +82,15 @@ def testSARWind_using_s1IWDVsafe_meps_filenames(sarIW_SAFE, meps): """ w = SARWind(sarIW_SAFE, meps) assert type(w) == SARWind + +@pytest.mark.sardata +def testSARData_from_NBS(): + """ Test using CSW for reading Sentinel-1 data. + This test requres access to https://nbs.csw.met.no/csw + """ + bbox = [-10, 75, 40, 85] + stop = datetime(2022, 1, 8, 5, 30, 59).replace(tzinfo=pytz.utc) + start = stop - timedelta(days=1) + + sw = SARData(bbox=bbox, start=start, stop=stop, kw_name='S1A*') + assert type(sw) == SARData From 528ab8420b52678935eb31c71492b9c7b57ff000 Mon Sep 17 00:00:00 2001 From: froded Date: Fri, 10 Feb 2023 14:45:40 +0100 Subject: [PATCH 3/4] Added owslib to support csw search for Sentinel-1 data --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9db5e57..ea5dc05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ scipy six urllib3 xdg +owslib From 8c46d3812d096cb16975510d1768f97a0441dc81 Mon Sep 17 00:00:00 2001 From: froded Date: Fri, 10 Feb 2023 14:47:52 +0100 Subject: [PATCH 4/4] Corrected som errors --- sardata/sardataNBS.py | 102 ++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 63 deletions(-) diff --git a/sardata/sardataNBS.py b/sardata/sardataNBS.py index 6059652..ab27b08 100644 --- a/sardata/sardataNBS.py +++ b/sardata/sardataNBS.py @@ -1,52 +1,47 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- +""" License: This file is part of https://github.com/metno/met-sar-vind + met-sar-vind is licensed under the Apache-2.0 license + (https://github.com/metno/met-sar-vind/blob/main/LICENSE). """ -Created on Thu Feb 9 09:32:05 2023 - -@author: froded -""" - from owslib import fes from owslib.fes import SortBy, SortProperty from owslib.csw import CatalogueServiceWeb - class SARData(): """ - A class for getting Sentinel-1 netCDF data from the Norwegian Ground Segment (NBS) + A class for getting Sentinel-1 netCDF data from the Norwegian Ground Segment (NBS) - Parameters - ----------- - endpoint : str + Parameters + ----------- + endpoint : str URL to NBS. Default: https://nbs.csw.met.no/csw - bbox : int list - A boudary box for search area speified in latitude and longitude + bbox : int list + A boudary box for search area speified in latitude and longitude bbox = [lon_min, lat_min, lon_max, lat_max] - + Example: >>> bbox = [-10, 75, 40, 85] - - start : datetime object - Specify the start time of the range to serchthe - - stop : datetime object - Specify the stop time of the range to serchthe - + + start : datetime object + Specify the start time of the range to serchthe + + stop : datetime object + Specify the stop time of the range to serchthe + Example: >>> from datetime import datetime, timedelta >>> stop = datetime(2010, 1, 1, 12, 30, 59).replace(tzinfo=pytz.utc) >>> start = stop - timedelta(days=7) - - kw_names : str - A search string filter to limit the result of a search. + + kw_names : str + A search string filter to limit the result of a search. Example: - >>> kw_name='S1A*' - + >>> kw_name='S1A*' + """ - def __init__(self, endpoint='https://nbs.csw.met.no/csw', bbox=None, start=None, stop=None, kw_names=None, crs="urn:ogc:def:crs:OGC:1.3:CRS84", *args, **kwargs): - + def __init__(self, endpoint='https://nbs.csw.met.no/csw', bbox=None, start=None, stop=None, + kw_names=None, crs='urn:ogc:def:crs:OGC:1.3:CRS84', *args, **kwargs): constraints = [] csw = None while csw is None: @@ -55,44 +50,39 @@ def __init__(self, endpoint='https://nbs.csw.met.no/csw', bbox=None, start=None, csw = CatalogueServiceWeb(endpoint, timeout=60) except: pass - + if kw_names: kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText") - or_filt = fes.Or([fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in kw_names]) + or_filt = fes.Or([ + fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in kw_names]) constraints.append(or_filt) - - - - if all(v is not None for v in [start, stop]): + + if all(v is not None for v in [start, stop]): begin, end = self._fes_date_filter(start, stop) constraints.append(begin) constraints.append(end) - + if bbox: bbox_crs = fes.BBox(bbox, crs=crs) constraints.append(bbox_crs) if len(constraints) >= 2: - filter_list = [ - fes.And( - constraints - ) - ] + filter_list = [fes.And(constraints)] else: filter_list = constraints - + self._get_csw_records(csw, filter_list, pagesize=2, maxrecords=10) self.csw = csw url_opendap = [] - + for key, value in list(csw.records.items()): for ref in value.references: if ref['scheme'] == 'OPENDAP:OPENDAP': url_opendap.append(ref['url']) self.url_opendap = url_opendap - - - - + + + + def _get_csw_records(self, csw, filter_list, pagesize=2, maxrecords=10): """Iterate `maxrecords`/`pagesize` times until the requested value in `maxrecords` is reached. @@ -116,17 +106,14 @@ def _get_csw_records(self, csw, filter_list, pagesize=2, maxrecords=10): if startposition >= maxrecords: break csw.records.update(csw_records) - - - - + def _fes_date_filter(self, start, stop, constraint="within"): """ Take datetime-like objects and returns a fes filter for date range (begin and end inclusive). NOTE: Truncates the minutes!!! """ - + start = start.strftime("%Y-%m-%d %H:00") stop = stop.strftime("%Y-%m-%d %H:00") if constraint == "overlaps": @@ -146,14 +133,3 @@ def _fes_date_filter(self, start, stop, constraint="within"): else: raise NameError("Unrecognized constraint {}".format(constraint)) return begin, end - - - - - - - - - - - \ No newline at end of file