Merge branch 'dev' into port-example-args-to-yamls

CHANGELOG.md

@@ -8,13 +8,15 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 ### Added
-* [656](https://github.com/dbekaert/RAiDER/pull/656) - example run configuration files available through `raider.py --generate_config <example_name>`.
+* [656](https://github.com/dbekaert/RAiDER/pull/656) - Example run configuration files available through `raider.py --generate_config <example_name>`.
 ### Changed
-* [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`.
 * [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
-* [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and add unit tests and bug fixes.
-* [662](https://github.com/dbekaert/RAiDER/pull/662) - Ensures dem-stitcher to be >= v2.5.6, which updates the url for reading the Geoid EGM 2008.
-* [661](https://github.com/dbekaert/RAiDER/pull/661) - Fix bug in raiderDownloadGNSS, remove call to scipy.sum, and add unit tests
+* [662](https://github.com/dbekaert/RAiDER/pull/662) - Bumped dem-stitcher to >= v2.5.6, which updates the URL for reading the Geoid EGM 2008.
+### Fixed
+* [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and added unit tests and bug fixes.
+* [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`.
+* [658](https://github.com/dbekaert/RAiDER/pull/658) - Fixed opaque error message if a GUNW file is not produced while HyP3 independently against a previous INSAR_ISCE.
+* [661](https://github.com/dbekaert/RAiDER/pull/661) - Fixed bug in raiderDownloadGNSS, removed call to scipy.sum, and added unit tests.
 
 ## [0.5.1]
 ### Changed

environment.yml

@@ -67,3 +67,5 @@ dependencies:
  - jupyter_contrib_nbextensions
  - jupyterlab
  - wand
+ # For development
+ - boto3-stubs

test/test_raises_for_missing_gunw.py

@@ -0,0 +1,83 @@
+'''
+Regression tests for issue #648:
+Bad error message when GUNW file missing in S3 bucket
+
+Program should raise an error if the GUNW product file, metadata file,
+or browse image is missing that clearly explains what went wrong, as opposed to
+a generic error message resulting from a side effect of the error.
+'''
+from contextlib import contextmanager
+from typing import List
+
+import pytest
+from test import TEST_DIR
+
+import shutil
+from tempfile import TemporaryDirectory
+from pathlib import Path
+import RAiDER.aws
+import RAiDER.cli.raider
+
+EXAMPLE_GUNW_PATH = Path(TEST_DIR) / 'gunw_test_data/S1-GUNW-D-R-059-tops-20230320_20220418-180300-00179W_00051N-PP-c92e-v2_0_6.nc'
+EXAMPLE_JSON_DATA_PATH = Path(TEST_DIR) / 'gunw_test_data/S1-GUNW-A-R-064-tops-20210723_20210711-015001-35393N_33512N-PP-6267-v2_0_4.json'
+
+
+@pytest.fixture
+def iargs() -> List[str]:
+    return [
+        '--bucket', 'dummy-bucket',
+        '--input-bucket-prefix', 'dummy-input-prefix',
+        '--weather-model', 'ERA5',
+    ]
+
+@contextmanager
+def make_gunw_path():
+    with TemporaryDirectory() as tempdir:
+        shutil.copy(EXAMPLE_GUNW_PATH, tempdir)
+        yield Path(tempdir) / Path(EXAMPLE_GUNW_PATH).name
+
+@contextmanager
+def make_json_data_path():
+    with TemporaryDirectory() as tempdir:
+        shutil.copy(EXAMPLE_JSON_DATA_PATH, tempdir)
+        yield Path(tempdir) / Path(EXAMPLE_JSON_DATA_PATH).name
+
+
+# Patch aws.get_s3_file to produce None then check for the correct error message
+def test_missing_product_file(mocker, iargs):
+    side_effect = [
+        None,  # GUNW product file
+        # program should fail
+    ]
+    mocker.patch('RAiDER.aws.get_s3_file', side_effect=side_effect)
+    with pytest.raises(ValueError) as excinfo:
+        RAiDER.cli.raider.calcDelaysGUNW(iargs)
+    assert "GUNW product file could not be found" in str(excinfo.value)
+
+
+# Patch aws.get_s3_file to produce None then check for the correct error message
+def test_missing_metadata_file(mocker, iargs):
+    with make_gunw_path() as gunw_path:
+        side_effect = [
+            gunw_path,  # GUNW product file
+            None,  # GUNW metadata file
+            # program should fail
+        ]
+        mocker.patch('RAiDER.aws.get_s3_file', side_effect=side_effect)
+        with pytest.raises(ValueError) as excinfo:
+            RAiDER.cli.raider.calcDelaysGUNW(iargs)
+        assert "GUNW metadata file could not be found" in str(excinfo.value)
+
+
+def test_missing_browse_image(mocker, iargs):
+    with make_gunw_path() as gunw_path, make_json_data_path() as json_data_path:
+        side_effect = [
+            gunw_path,  # GUNW product file
+            json_data_path,  # GUNW metadata file
+            None,  # GUNW browse image
+            # program should fail
+        ]
+        mocker.patch('RAiDER.aws.get_s3_file', side_effect=side_effect)
+        with pytest.raises(ValueError) as excinfo:
+            RAiDER.cli.raider.calcDelaysGUNW(iargs)
+        assert "GUNW browse image could not be found" in str(excinfo.value)

tools/RAiDER/aws.py

@@ -1,6 +1,6 @@
+from typing import Optional, Union
 from mimetypes import guess_type
 from pathlib import Path
-from typing import Union
 
 import boto3
 
@@ -9,7 +9,7 @@
 S3_CLIENT = boto3.client('s3')
 
 
-def get_tag_set() -> dict:
+def get_tag_set():
     tag_set = {
         'TagSet': [
             {
@@ -28,7 +28,7 @@ def get_content_type(file_location: Union[Path, str]) -> str:
     return content_type
 
 
-def upload_file_to_s3(path_to_file: Union[str, Path], bucket: str, prefix: str = ''):
+def upload_file_to_s3(path_to_file: Union[str, Path], bucket: str, prefix: str = '') -> None:
     path_to_file = Path(path_to_file)
     key = str(Path(prefix) / path_to_file)
     extra_args = {'ContentType': get_content_type(key)}
@@ -41,13 +41,15 @@ def upload_file_to_s3(path_to_file: Union[str, Path], bucket: str, prefix: str =
     S3_CLIENT.put_object_tagging(Bucket=bucket, Key=key, Tagging=tag_set)
 
 
-def get_s3_file(bucket_name, bucket_prefix, file_type: str):
-    result = S3_CLIENT.list_objects_v2(Bucket=bucket_name, Prefix=bucket_prefix)
+def get_s3_file(bucket_name: str, bucket_prefix: str, file_type: str) -> Optional[str]:
+    result = S3_CLIENT.list_objects_v2(
+        Bucket=bucket_name,
+        Prefix=bucket_prefix
+    )
     for s3_object in result['Contents']:
         key = s3_object['Key']
         if key.endswith(file_type):
             file_name = Path(key).name
             logger.info(f'Downloading s3://{bucket_name}/{key} to {file_name}')
             S3_CLIENT.download_file(bucket_name, key, file_name)
             return file_name
-

tools/RAiDER/cli/raider.py

@@ -552,6 +552,11 @@ def calcDelaysGUNW(iargs: list[str] = None) -> xr.Dataset:
         # only use GUNW ID for checking if HRRR available
         iargs.file = aws.get_s3_file(
             iargs.bucket, iargs.input_bucket_prefix, '.nc')
+        if iargs.file is None:
+            raise ValueError(
+                'GUNW product file could not be found at'
+                f's3://{iargs.bucket}/{iargs.input_bucket_prefix}'
+            )
         if iargs.weather_model == 'HRRR' and (iargs.interpolate_time == 'azimuth_time_grid'):
             file_name_str = str(iargs.file)
             gunw_nc_name = file_name_str.split('/')[-1]
@@ -569,6 +574,11 @@ def calcDelaysGUNW(iargs: list[str] = None) -> xr.Dataset:
             return
         json_file_path = aws.get_s3_file(
             iargs.bucket, iargs.input_bucket_prefix, '.json')
+        if json_file_path is None:
+            raise ValueError(
+                'GUNW metadata file could not be found at'
+                f's3://{iargs.bucket}/{iargs.input_bucket_prefix}'
+            )
         json_data = json.load(open(json_file_path))
         json_data['metadata'].setdefault(
             'weather_model', []).append(iargs.weather_model)
@@ -577,6 +587,11 @@ def calcDelaysGUNW(iargs: list[str] = None) -> xr.Dataset:
         # also get browse image -- if RAiDER is running in its own HyP3 job, the browse image will be needed for ingest
         browse_file_path = aws.get_s3_file(
             iargs.bucket, iargs.input_bucket_prefix, '.png')
+        if browse_file_path is None:
+            raise ValueError(
+                'GUNW browse image could not be found at'
+                f's3://{iargs.bucket}/{iargs.input_bucket_prefix}'
+            )
 
     elif not iargs.file:
         raise ValueError('Either argument --file or --bucket must be provided')