Merge pull request #656 from garlic-os/port-example-args-to-yamls

Re-add example run configurations

CHANGELOG.md

@@ -7,13 +7,16 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+* [656](https://github.com/dbekaert/RAiDER/pull/656) - Example run configuration files available through `raider.py --generate_config <example_name>`.
+### Changed
+* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
+* [662](https://github.com/dbekaert/RAiDER/pull/662) - Bumped dem-stitcher to >= v2.5.6, which updates the URL for reading the Geoid EGM 2008.
 ### Fixed
 * [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and added unit tests and bug fixes.
-* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
 * [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`.
 * [658](https://github.com/dbekaert/RAiDER/pull/658) - Fixed opaque error message if a GUNW file is not produced while HyP3 independently against a previous INSAR_ISCE.
 * [661](https://github.com/dbekaert/RAiDER/pull/661) - Fixed bug in raiderDownloadGNSS, removed call to scipy.sum, and added unit tests.
-* [662](https://github.com/dbekaert/RAiDER/pull/662) - Ensures dem-stitcher to be >= v2.5.6, which updates the url for reading the Geoid EGM 2008.
 
 ## [0.5.1]
 ### Changed

README.md

@@ -85,8 +85,8 @@ RAiDER has the ability to download weather models from third-parties; some of wh
 ## 3. Running RAiDER and Documentation
 
 For detailed documentation, examples, and Jupyter notebooks see the [RAiDER-docs repository](https://github.com/dbekaert/RAiDER-docs).
-We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions).
-``` raider.py -h ``` provides a help menu and list of example commands to get started.
+We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions).  
+``` raider.py -h ``` provides a help menu and set of example run configurations to get started.
 The RAiDER scripts are highly modularized in Python and allows for building your own processing workflow.
 
 ------

test/__init__.py

@@ -32,25 +32,30 @@ def pushd(dir):
 def update_yaml(dct_cfg:dict, dst:str='temp.yaml'):
     """ Write a new yaml file from a dictionary.
 
-    Updates parameters in the default 'raider.yaml' file.
+    Updates parameters in the default 'template.yaml' file.
     Each key:value pair will in 'dct_cfg' will overwrite that in the default
     """
     import RAiDER, yaml
 
-    template_file = os.path.join(
-                    os.path.dirname(RAiDER.__file__), 'cli', 'raider.yaml')
+    run_config_path = os.path.join(
+        os.path.dirname(RAiDER.__file__),
+        'cli',
+        'examples',
+        'template',
+        'template.yaml'
+    )
 
-    with open(template_file, 'r') as f:
+    with open(run_config_path, 'r') as f:
         try:
             params = yaml.safe_load(f)
         except yaml.YAMLError as exc:
             print(exc)
-            raise ValueError(f'Something is wrong with the yaml file {template_file}')
+            raise ValueError(f'Something is wrong with the yaml file {run_config_path}')
 
     params = {**params, **dct_cfg}
 
     with open(dst, 'w') as fh:
-        yaml.safe_dump(params, fh,  default_flow_style=False)
+        yaml.safe_dump(params, fh, default_flow_style=False)
 
     return dst
 

test/test_datelist.py

@@ -2,7 +2,7 @@
 import os
 import shutil
 from test import TEST_DIR, WM, update_yaml
-from RAiDER.cli.raider import read_template_file
+from RAiDER.cli.raider import read_run_config_file
 
 def test_datelist():
     SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist')
@@ -27,7 +27,7 @@ def test_datelist():
       }
 
     cfg  = update_yaml(dct_group, 'temp.yaml')
-    param_dict = read_template_file(cfg)
+    param_dict = read_run_config_file(cfg)
     assert param_dict['date_list'] == true_dates
 
 
@@ -52,5 +52,5 @@ def test_datestep():
       }
 
     cfg  = update_yaml(dct_group, 'temp.yaml')
-    param_dict = read_template_file(cfg)
+    param_dict = read_run_config_file(cfg)
     assert param_dict['date_list'] == true_dates

test/test_generate_config.py

@@ -0,0 +1,73 @@
+'''
+Tests the functionality of `raider.py --generate_config <example_file>`.
+For any given example file name, the script should produce that run
+configuration file in the current working directory, along wihh any
+corresponding data files.
+If such a file already exists, the script should prompt the user to
+confirm overwriting the file.
+'''
+import pytest
+
+import os
+import subprocess
+import tempfile
+from contextlib import contextmanager
+
+
+@contextmanager
+def cd_to_temp_dir():
+    old_pwd = os.getcwd()
+    try:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            os.chdir(tmpdir)
+            yield
+    finally:
+        os.chdir(old_pwd)
+
+@pytest.mark.parametrize(
+    'name,data_files',
+    [
+        ('template', []),
+        ('example_LA_bbox', []),
+        ('example_LA_GNSS', ['example_LA_GNSS.csv']),
+        ('example_UK_isce', ['example_UK_isce-S1B_OPER_AUX_POEORB_OPOD_20211122T112354_V20211101T225942_20211103T005942.EOF']),
+    ]
+)
+def test_generate_config(name, data_files):
+    with cd_to_temp_dir():
+        subprocess.run(['raider.py', '--generate_config', name])
+        assert os.path.exists(f'{name}.yaml')
+        for data_file_name in data_files:
+            assert os.path.exists(data_file_name)
+
+
+def test_confirm_overwrite_yes():
+    with cd_to_temp_dir():
+        with open('template.yaml', 'w') as f:
+            f.write('overwrite me')
+        process = subprocess.Popen(
+            ['raider.py', '--generate_config', 'template'],
+            stdin=subprocess.PIPE,
+        )
+        process.stdin.write(b'y\n')
+        process.stdin.close()
+        process.wait()
+        assert os.path.exists('template.yaml')
+        with open('template.yaml') as f:
+            assert f.read() != 'overwrite me'
+
+
+def test_confirm_overwrite_no():
+    with cd_to_temp_dir():
+        with open('template.yaml', 'w') as f:
+            f.write("don't overwrite me")
+        process = subprocess.Popen(
+            ['raider.py', '--generate_config', 'template'],
+            stdin=subprocess.PIPE,
+        )
+        process.stdin.write(b'n\n')
+        process.stdin.close()
+        process.wait()
+        assert os.path.exists('template.yaml')
+        with open('template.yaml') as f:
+            assert f.read() == "don't overwrite me"

test/test_intersect.py

@@ -19,7 +19,7 @@ def test_cube_intersect(wm):
     time       ='13:52:45'
     # f_lat, f_lon = makeLatLonGrid([S, N, W, E], 'LA', SCENARIO_DIR, 0.25)
 
-    ## make the template file
+    ## make the run config file
     grp = {
             'date_group': {'date_start': date},
             'time_group': {'time': time, 'interpolate_time': 'none'},
@@ -35,7 +35,7 @@ def test_cube_intersect(wm):
             'verbose': False,
         }
 
-    ## generate the default template file and overwrite it with new parms
+    ## generate the default run config file and overwrite it with new parms
     cfg  = update_yaml(grp, 'temp.yaml')
 
     ## run raider and intersect
@@ -68,7 +68,7 @@ def test_gnss_intersect(wm):
     date       = 20200130
     time       ='13:52:45'
 
-    ## make the template file
+    ## make the run config file
     grp = {
             'date_group': {'date_start': date},
             'time_group': {'time': time, 'interpolate_time': 'none'},
@@ -81,7 +81,7 @@ def test_gnss_intersect(wm):
             'verbose': False,
         }
 
-    ## generate the default template file and overwrite it with new parms
+    ## generate the default run config file and overwrite it with new parms
     cfg  = update_yaml(grp)
 
     ## run raider and intersect

test/test_slant.py

@@ -15,7 +15,7 @@ def test_slant_proj(weather_model_name):
     date       = 20200130
     time       ='13:52:45'
 
-    ## make the template file
+    ## make the run config file
     grp = {
             'date_group': {'date_start': date},
             'height_group': {'height_levels': [0, 100, 500, 1000]},
@@ -32,7 +32,7 @@ def test_slant_proj(weather_model_name):
            }
         }
 
-    ## generate the default template file and overwrite it with new parms
+    ## generate the default run config file and overwrite it with new parms
     cfg  = update_yaml(grp, 'temp.yaml')
 
     ## run raider and intersect
@@ -66,7 +66,7 @@ def test_ray_tracing(weather_model_name):
     date       = 20200130
     time       ='13:52:45'
 
-    ## make the template file
+    ## make the run config file
     grp = {
             'date_group': {'date_start': date},
             'height_group': {'height_levels': [0, 100, 500, 1000]},
@@ -83,7 +83,7 @@ def test_ray_tracing(weather_model_name):
            }
         }
 
-    ## generate the default template file and overwrite it with new parms
+    ## generate the default run config file and overwrite it with new parms
     cfg  = update_yaml(grp, 'temp.yaml')
 
     ## run raider and intersect

test/test_temporal_interpolate.py

@@ -33,7 +33,7 @@ def test_cube_timemean():
     ## run raider without interpolation for two exact weather model times
     for hr in [hr1, hr2]:
         grp['time_group'].update({'time': f'{hr}:00:00'})
-        ## generate the default template file and overwrite it with new parms
+        ## generate the default run config file and overwrite it with new parms
         cfg  = update_yaml(grp)
 
         ## run raider for the default date
@@ -95,7 +95,7 @@ def test_cube_weighting():
     ## run raider without interpolation for two exact weather model times
     for hr in [hr1, hr2]:
         grp['time_group'].update({'time': f'{hr}:00:00'})
-        ## generate the default template file and overwrite it with new parms
+        ## generate the default run config file and overwrite it with new parms
         cfg  = update_yaml(grp)
 
         ## run raider for the default date

tools/RAiDER/aria/prepFromGUNW.py

@@ -360,19 +360,24 @@ def make_cube(self):
 def update_yaml(dct_cfg:dict, dst:str='GUNW.yaml'):
     """ Write a new yaml file from a dictionary.
 
-    Updates parameters in the default 'raider.yaml' file.
+    Updates parameters in the default 'template.yaml' file.
     Each key:value pair will in 'dct_cfg' will overwrite that in the default
     """
 
-    template_file = os.path.join(
-                    os.path.dirname(RAiDER.__file__), 'cli', 'raider.yaml')
+    run_config_path = os.path.join(
+        os.path.dirname(RAiDER.__file__),
+        'cli',
+        'examples',
+        'template',
+        'template.yaml'
+    )
 
-    with open(template_file, 'r') as f:
+    with open(run_config_path, 'r') as f:
         try:
             params = yaml.safe_load(f)
         except yaml.YAMLError as exc:
             print(exc)
-            raise ValueError(f'Something is wrong with the yaml file {template_file}')
+            raise ValueError(f'Something is wrong with the yaml file {run_config_path}')
 
     params = {**params, **dct_cfg}
 

tools/RAiDER/cli/__init__.py

@@ -8,37 +8,37 @@ class AttributeDict(dict):
 
 DEFAULT_DICT = AttributeDict(
     dict(
-            look_dir='right',
-            date_start=None,
-            date_end=None,
-            date_step=None,
-            date_list=None,
-            time=None,
-            end_time=None,
-            weather_model=None,
-            lat_file=None,
-            lon_file=None,
-            station_file=None,
-            bounding_box=None,
-            geocoded_file=None,
-            dem=None,
-            use_dem_latlon=False,
-            height_levels=None,
-            height_file_rdr=None,
-            ray_trace=False,
-            zref=_ZREF,
-            cube_spacing_in_m=_CUBE_SPACING_IN_M,
-            los_file=None,
-            los_convention='isce',
-            los_cube=None,
-            orbit_file=None,
-            verbose=True,
-            raster_format='GTiff',
-            file_format='GTiff',
-            download_only=False,
-            output_directory='.',
-            weather_model_directory=None,
-            output_projection='EPSG:4326',
-            interpolate_time='center_time',
-        )
+        look_dir='right',
+        date_start=None,
+        date_end=None,
+        date_step=None,
+        date_list=None,
+        time=None,
+        end_time=None,
+        weather_model=None,
+        lat_file=None,
+        lon_file=None,
+        station_file=None,
+        bounding_box=None,
+        geocoded_file=None,
+        dem=None,
+        use_dem_latlon=False,
+        height_levels=None,
+        height_file_rdr=None,
+        ray_trace=False,
+        zref=_ZREF,
+        cube_spacing_in_m=_CUBE_SPACING_IN_M,
+        los_file=None,
+        los_convention='isce',
+        los_cube=None,
+        orbit_file=None,
+        verbose=True,
+        raster_format='GTiff',
+        file_format='GTiff',
+        download_only=False,
+        output_directory='.',
+        weather_model_directory=None,
+        output_projection='EPSG:4326',
+        interpolate_time='center_time',
     )
+)