more informative surge interpolation errors, and catch failed interpo…

…lation

pyCIAM/run.py

@@ -332,6 +332,24 @@ def calc_costs(
             # interpolation with 0's
             surge_noadapt = []
             surge = []
+
+            def _check_vals(rh_diff_arr, lslr_arr, seg):
+                error_str = (
+                    "{0} value is {1} than {2} in storm damage lookup "
+                    f"table for segment {seg}. Please investigate why this is. You may "
+                    "need to re-generate the surge lookup table (or perhaps the `refA` "
+                    "(initial adaptation) table if it was generated for a different "
+                    "set of SLR scenarios or years."
+                )
+                if rh_diff_arr.max() > this_surge_lookup.rh_diff.max():
+                    raise ValueError(error_str.format("rh_diff", "higher", "maximum"))
+                if rh_diff_arr.min() < this_surge_lookup.rh_diff.min():
+                    raise ValueError(error_str.format("rh_diff", "lower", "minimum"))
+                if lslr_arr.max() > this_surge_lookup.lslr.max():
+                    raise ValueError(error_str.format("lslr", "higher", "maximum"))
+                if lslr_arr.min() < this_surge_lookup.lslr.min():
+                    raise ValueError(error_str.format("lslr", "lower", "minimum"))
+
             for seg in inputs.seg.values:
                 this_surge_lookup = (
                     surge_lookup.sel(seg=seg)
@@ -342,38 +360,47 @@ def calc_costs(
                 lslr_too_low = lslr.sel(seg=seg) < this_surge_lookup.lslr.min()
                 if this_surge_lookup.sum() == 0:
                     continue
+
+                this_rh_diff_noadapt = rh_diff_noadapt.sel(seg=seg, drop=True)
+                this_lslr = lslr.sel(seg=seg, drop=True)
+                _check_vals(this_rh_diff_noadapt, this_lslr, seg)
+
                 this_surge_noadapt = (
                     this_surge_lookup.sel(adapttype="retreat", drop=True)
                     .interp(
-                        lslr=lslr.sel(seg=seg),
-                        rh_diff=rh_diff_noadapt.sel(seg=seg),
+                        lslr=this_lslr,
+                        rh_diff=this_rh_diff_noadapt,
                         assume_sorted=True,
                     )
                     .reset_coords(drop=True)
                     .expand_dims(seg=[seg])
                 )
 
                 # ensure nans are only at the beginning
-                assert (this_surge_noadapt.notnull() | lslr_too_low).all()
+                assert (this_surge_noadapt.notnull() | lslr_too_low).all(), seg
 
                 surge_noadapt.append(this_surge_noadapt.fillna(0))
 
                 surge_adapt = []
+
+                this_rh_diff_adapt = rh_diff.sel(seg=seg, drop=True)
+                _check_vals(this_rh_diff_noadapt, this_lslr, seg)
+
                 for adapttype in this_surge_lookup.adapttype.values:
                     this_surge_adapt = (
                         this_surge_lookup.sel(adapttype=adapttype)
                         .interp(
-                            lslr=lslr.sel(seg=seg),
-                            rh_diff=rh_diff.sel(
-                                adapttype=adapttype, seg=seg, drop=True
+                            lslr=this_lslr,
+                            rh_diff=this_rh_diff_adapt.sel(
+                                adapttype=adapttype, drop=True
                             ),
                             assume_sorted=True,
                         )
                         .reset_coords(drop=True)
                     )
 
                     # ensure nans are only at the beginning
-                    assert (this_surge_adapt.notnull() | lslr_too_low).all()
+                    assert (this_surge_adapt.notnull() | lslr_too_low).all(), seg
 
                     surge_adapt.append(this_surge_adapt.fillna(0))
                 surge.append(
@@ -1157,7 +1184,7 @@ def execute_pyciam(
                 storage_options=storage_options,
             )
     # block on this calculation
-    wait(surge_futs)
+    client.gather(surge_futs)
 
     ###############################
     # define temporary output store
@@ -1380,7 +1407,7 @@ def execute_pyciam(
     ###############################
     # Rechunk and save final
     ###############################
-    wait(ciam_futs_2.tolist())
+    client.gather(ciam_futs_2.tolist())
     assert [f.status == "finished" for f in ciam_futs_2.tolist()]
     client.cancel(ciam_futs_2)
     del ciam_futs_2