Patch (#134)

* Fix square cell assumption

* Fix receiver hanging

* Fixed KeyboardInterrupt error

* Update changelog

* Adjusted testdata for rounding erros

* Updated tests

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -12,13 +12,8 @@ repos:
       - id: debug-statements
       - id: mixed-line-ending
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.2
+    rev: v0.6.9
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
-  # - repo: https://github.com/python-jsonschema/check-jsonschema
-  #   rev: 0.24.0
-  #   hooks:
-  #     - id: check-github-workflows
-  #     - id: check-github-actions

.testdata/create_test_data.py

@@ -41,16 +41,42 @@ def create_exposure_dbase():
             f.write(f"{n+1},area,0,0,{dmc},{(n+1)*1000}\n")
 
 
+def create_exposure_dbase_missing():
+    """_summary_."""
+    with open(Path(p, "exposure", "spatial_missing.csv"), "w") as f:
+        f.write("extract_method,ground_flht,ground_elevtn,")
+        f.write("fn_damage_structure,max_damage_structure\n")
+        for n in range(5):
+            if (n + 1) % 2 != 0:
+                dmc = "struct_1"
+            else:
+                dmc = "struct_2"
+            f.write(f"area,0,0,{dmc},{(n+1)*1000}\n")
+
+
+def create_exposure_dbase_partial():
+    """_summary_."""
+    with open(Path(p, "exposure", "spatial_partial.csv"), "w") as f:
+        f.write("object_id,extract_method,ground_flht,ground_elevtn,")
+        f.write("fn_damage_structure,fn_damage_content,max_damage_structure\n")
+        for n in range(5):
+            if (n + 1) % 2 != 0:
+                dmc = "struct_1"
+            else:
+                dmc = "struct_2"
+            f.write(f"{n+1},area,0,0,{dmc},{dmc},{(n+1)*1000}\n")
+
+
 def create_exposure_geoms():
     """_summary_."""
     geoms = (
         "POLYGON ((4.355 52.045, 4.355 52.035, 4.365 52.035, \
 4.365 52.045, 4.355 52.045))",
         "POLYGON ((4.395 52.005, 4.395 51.975, 4.415 51.975, \
 4.415 51.985, 4.405 51.985, 4.405 52.005, 4.395 52.005))",
-        "POLYGON ((4.365 51.960, 4.375 51.990, 4.385 51.960, 4.365 51.960))",
-        "POLYGON ((4.410 52.030, 4.440 52.030, 4.435 52.010, \
-4.415 52.010, 4.410 52.030))",
+        "POLYGON ((4.365 51.9605, 4.375 51.9895, 4.385 51.9605, 4.365 51.9605))",
+        "POLYGON ((4.4105 52.0295, 4.4395 52.0295, 4.435 52.0105, \
+4.415 52.0105, 4.4105 52.0295))",
     )
     srs = osr.SpatialReference()
     srs.ImportFromEPSG(4326)
@@ -276,6 +302,45 @@ def create_hazard_map():
     dr = None
 
 
+def create_hazard_map_highres():
+    """_summary_."""
+    srs = osr.SpatialReference()
+    srs.ImportFromEPSG(4326)
+    dr = gdal.GetDriverByName("netCDF")
+    src = dr.Create(
+        str(Path(p, "hazard", "event_map_highres.nc")),
+        100,
+        100,
+        1,
+        gdal.GDT_Float32,
+    )
+    gtf = (
+        4.35,
+        0.001,
+        0.0,
+        52.05,
+        0.0,
+        -0.001,
+    )
+    src.SetSpatialRef(srs)
+    src.SetGeoTransform(gtf)
+
+    band = src.GetRasterBand(1)
+    data = zeros((100, 100))
+    oneD = tuple(range(100))
+    for x, y in product(oneD, oneD):
+        data[x, y] = 3.6 - ((x + y) * 0.02)
+    band.WriteArray(data)
+
+    band.FlushCache()
+    src.FlushCache()
+
+    srs = None
+    band = None
+    src = None
+    dr = None
+
+
 def create_risk_map():
     """_summary_."""
     srs = osr.SpatialReference()
@@ -396,6 +461,18 @@ def create_settings_geom():
     with open(Path(p, "geom_risk_2g.toml"), "wb") as f:
         tomli_w.dump(doc_r2g, f)
 
+    missing_hazard = copy.deepcopy(doc)
+    del missing_hazard["hazard"]["file"]
+
+    with open(Path(p, "missing_hazard.toml"), "wb") as f:
+        tomli_w.dump(missing_hazard, f)
+
+    missing_models = copy.deepcopy(doc)
+    del missing_models["exposure"]["geom"]["file1"]
+
+    with open(Path(p, "missing_models.toml"), "wb") as f:
+        tomli_w.dump(missing_models, f)
+
 
 def create_settings_grid():
     """_summary_."""
@@ -438,6 +515,12 @@ def create_settings_grid():
     with open(Path(p, "grid_risk.toml"), "wb") as f:
         tomli_w.dump(doc_r, f)
 
+    doc_u = copy.deepcopy(doc)
+    doc_u["hazard"]["file"] = "hazard/event_map_highres.nc"
+
+    with open(Path(p, "grid_unequal.toml"), "wb") as f:
+        tomli_w.dump(doc_u, f)
+
 
 def create_vulnerability():
     """_summary_."""
@@ -463,11 +546,14 @@ def log_base(b, x):
 if __name__ == "__main__":
     create_dbase_stucture()
     create_exposure_dbase()
+    create_exposure_dbase_missing()
+    create_exposure_dbase_partial()
     create_exposure_geoms()
     create_exposure_geoms_2()
     create_exposure_geoms_3()
     create_exposure_grid()
     create_hazard_map()
+    create_hazard_map_highres()
     create_risk_map()
     create_settings_geom()
     create_settings_grid()

docs/changelog.qmd

@@ -6,8 +6,13 @@ title: "What's new?"
 These are the unreleased changes of Delft-FIAT.
 
 ### Added
+- 'Normal' exit when keyboard interrupt is triggered over cli
+- Support for grids with non square cells
 
 ### Changed
+- Fixed hanging issue with the mp-logging receiver when erroring
+- Fixed square cell assumption when mapping world coordinates to pixel coordinates
+- Support capitalized entries in the settings toml (again; it was disabled)
 
 ### Deprecated
 

pyproject.toml

@@ -154,28 +154,22 @@ ignore_errors = true
 [tool.coverage.html]
 directory = ".cov"
 
-## Linting stuff
-[tool.black]
-line-length = 88
-target-version = ['py311']
-
 [tool.ruff]
 line-length = 88
+exclude = ["docs"]
 
-# enable pydocstyle (E), pyflake (F) and isort (I), pytest-style (PT)
+[tool.ruff.lint]
 select = ["E", "F", "I", "PT", "D"]
-ignore-init-module-imports = true
 ignore = ["B904", "D105", "D211", "D213", "D301", "E712", "E741"]
-exclude = ["docs"]
 
-[tool.ruff.per-file-ignores]
-"test/**" = ["D103", "D100", "D104"]
+[tool.ruff.lint.per-file-ignores]
+"test/**" = ["D100", "D101", "D102", "D103", "D104"]
 "test/conftest.py" = ["E402"]
 "src/fiat/__init__.py" = ["E402", "F401", "F403"]
 "src/fiat/cli/__init__.py" = ["F403"]
 "src/fiat/gis/__init__.py" = ["F403"]
 "src/fiat/methods/__init__.py" = ["F403"]
 "src/fiat/models/__init__.py" = ["F403"]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "numpy"

src/fiat/cfg.py

@@ -12,10 +12,10 @@
     check_config_grid,
 )
 from fiat.util import (
-    create_hidden_folder,
     flatten_dict,
     generic_folder_check,
     generic_path_check,
+    get_module_attr,
 )
 
 
@@ -51,10 +51,13 @@ def __init__(
         f.close()
 
         # Initial check for mandatory entries of the settings toml
+        extra_entries = get_module_attr(
+            f"fiat.methods.{self.get('global.type', 'flood')}", "MANDATORY_ENTRIES"
+        )
         check_config_entries(
             self.keys(),
             self.filepath,
-            self.path,
+            extra_entries,
         )
 
         # Set the cache size per GDAL object
@@ -74,9 +77,6 @@ def __init__(
                     self.path,
                 )
                 self[key] = path
-            else:
-                if isinstance(item, str):
-                    self[key] = item.lower()
 
         # Switch the build flag off
         self._build = False
@@ -106,16 +106,12 @@ def _create_dir(
         self,
         root: Path | str,
         path: Path | str,
-        hidden: bool = False,
     ):
         """_summary_."""
         _p = Path(path)
         if not _p.is_absolute():
             _p = Path(root, _p)
-        if hidden:
-            create_hidden_folder(_p)
-        else:
-            generic_folder_check(_p)
+        generic_folder_check(_p)
         return _p
 
     def _create_model_dirs(

src/fiat/check.py

@@ -16,16 +16,15 @@
 def check_config_entries(
     keys: tuple,
     path: Path,
-    parent: Path,
+    extra_entries: list,
 ):
     """_summary_."""
     _man_entries = [
         "output.path",
         "hazard.file",
         "hazard.risk",
-        "hazard.elevation_reference",
         "vulnerability.file",
-    ]
+    ] + extra_entries
 
     _check = [item in keys for item in _man_entries]
     if not all(_check):
@@ -143,7 +142,7 @@ def check_grid_exact(
 def check_internal_srs(
     source_srs: osr.SpatialReference,
     fname: str,
-    cfg_srs: osr.SpatialReference = None,
+    cfg_srs: str = None,
 ):
     """_summary_."""
     if source_srs is None and cfg_srs is None:
@@ -245,12 +244,13 @@ def check_hazard_subsets(
 
 ## Exposure
 def check_exp_columns(
+    index_col: str,
     columns: tuple | list,
     specific_columns: tuple | list = [],
 ):
     """_summary_."""
     _man_columns = [
-        "object_id",
+        index_col,
     ] + specific_columns
 
     _check = [item in columns for item in _man_columns]

src/fiat/cli/main.py

@@ -28,7 +28,7 @@
 ###############################################################
 
                 Fast Impact Assessment Tool
-                \u00A9 Deltares
+                \u00a9 Deltares
 
 """
 

src/fiat/cli/util.py

@@ -27,6 +27,9 @@ def run_log(
         func()
     except BaseException:
         t, v, tb = sys.exc_info()
-        logger.error(",".join([str(item) for item in v.args]))
+        msg = ",".join([str(item) for item in v.args])
+        if t is KeyboardInterrupt:
+            msg = "KeyboardInterrupt"
+        logger.error(msg)
         # Exit with code 1
         sys.exit(1)

src/fiat/gis/geom.py

@@ -74,7 +74,7 @@ def reproject(
     if not Path(str(out_dir)).is_dir():
         out_dir = gs.path.parent
 
-    fname = Path(out_dir, f"{gs.path.stem}_repr_fiat{gs.path.suffix}")
+    fname = Path(out_dir, f"{gs.path.stem}_repr{gs.path.suffix}")
 
     out_srs = osr.SpatialReference()
     out_srs.SetFromUserInput(crs)

src/fiat/gis/grid.py

@@ -70,8 +70,8 @@ def reproject(
     if not Path(str(out_dir)).is_dir():
         out_dir = gs.path.parent
 
-    fname_int = Path(out_dir, f"{gs.path.stem}_repr_fiat.tif")
-    fname = Path(out_dir, f"{gs.path.stem}_repr_fiat{gs.path.suffix}")
+    fname_int = Path(out_dir, f"{gs.path.stem}_repr.tif")
+    fname = Path(out_dir, f"{gs.path.stem}_repr{gs.path.suffix}")
 
     out_srs = osr.SpatialReference()
     out_srs.SetFromUserInput(dst_crs)

src/fiat/gis/overlay.py

@@ -85,7 +85,7 @@ def clip(
 
 
 def clip_weighted(
-    band: gdal.Band,
+    band: Grid,
     srs: osr.SpatialReference,
     gtf: tuple,
     ft: ogr.Feature,
@@ -142,7 +142,7 @@ def clip_weighted(
     pxWidth = int(lrX - ulX) + 1
     pxHeight = int(lrY - ulY) + 1
 
-    clip = band.ReadAsArray(ulX, ulY, pxWidth, pxHeight)
+    clip = band[ulX, ulY, pxWidth, pxHeight]
     # m = mask.ReadAsArray(ulX,ulY,pxWidth,pxHeight)
 
     # pts = geom.GetGeometryRef(0)
@@ -206,8 +206,8 @@ def pin(
     array
         A NumPy array containing one value.
     """
-    X, Y = world2pixel(gtf, *point)
+    x, y = world2pixel(gtf, *point)
 
-    value = band[X, Y, 1, 1]
+    value = band[x, y, 1, 1]
 
     return value[0]

src/fiat/gis/util.py

@@ -42,9 +42,10 @@ def world2pixel(
     ulX = gtf[0]
     ulY = gtf[3]
     xDist = gtf[1]
-    pixel = int((x - ulX) / xDist)
-    line = int((ulY - y) / xDist)
-    return (pixel, line)
+    yDist = gtf[5]
+    coorX = int((x - ulX) / xDist)
+    coorY = int((y - ulY) / yDist)
+    return (coorX, coorY)
 
 
 def pixel2world(