From c41df997fcf178def9ea8fd648081c5a2fce8534 Mon Sep 17 00:00:00 2001 From: dalmijn <92092029+dalmijn@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:59:01 +0200 Subject: [PATCH] Patch (#134) * Fix square cell assumption * Fix receiver hanging * Fixed KeyboardInterrupt error * Update changelog * Adjusted testdata for rounding erros * Updated tests --- .pre-commit-config.yaml | 9 +- .testdata/create_test_data.py | 92 +++++++++++++++++++- docs/changelog.qmd | 5 ++ pyproject.toml | 16 ++-- src/fiat/cfg.py | 16 ++-- src/fiat/check.py | 10 +-- src/fiat/cli/main.py | 2 +- src/fiat/cli/util.py | 5 +- src/fiat/gis/geom.py | 2 +- src/fiat/gis/grid.py | 4 +- src/fiat/gis/overlay.py | 8 +- src/fiat/gis/util.py | 7 +- src/fiat/io.py | 10 +-- src/fiat/log.py | 33 ++++++- src/fiat/methods/ead.py | 1 + src/fiat/methods/flood.py | 3 +- src/fiat/models/geom.py | 15 +++- src/fiat/models/worker_geom.py | 2 +- src/fiat/models/worker_grid.py | 1 + src/fiat/util.py | 36 ++------ test/conftest.py | 60 +++++++++---- test/test_checks.py | 153 +++++++++++++++++++++++++++++++++ test/test_gis.py | 78 ++++++++++++++--- test/test_model.py | 3 +- test/test_struct.py | 2 +- 25 files changed, 449 insertions(+), 124 deletions(-) create mode 100644 test/test_checks.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 08455ebb..8b47042c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -12,13 +12,8 @@ repos: - id: debug-statements - id: mixed-line-ending - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.2 + rev: v0.6.9 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - id: ruff-format - # - repo: https://github.com/python-jsonschema/check-jsonschema - # rev: 0.24.0 - # hooks: - # - id: check-github-workflows - # - id: check-github-actions diff --git a/.testdata/create_test_data.py b/.testdata/create_test_data.py index e3194d1d..57df18e6 100644 --- a/.testdata/create_test_data.py +++ b/.testdata/create_test_data.py @@ -41,6 +41,32 @@ def create_exposure_dbase(): f.write(f"{n+1},area,0,0,{dmc},{(n+1)*1000}\n") +def create_exposure_dbase_missing(): + """_summary_.""" + with open(Path(p, "exposure", "spatial_missing.csv"), "w") as f: + f.write("extract_method,ground_flht,ground_elevtn,") + f.write("fn_damage_structure,max_damage_structure\n") + for n in range(5): + if (n + 1) % 2 != 0: + dmc = "struct_1" + else: + dmc = "struct_2" + f.write(f"area,0,0,{dmc},{(n+1)*1000}\n") + + +def create_exposure_dbase_partial(): + """_summary_.""" + with open(Path(p, "exposure", "spatial_partial.csv"), "w") as f: + f.write("object_id,extract_method,ground_flht,ground_elevtn,") + f.write("fn_damage_structure,fn_damage_content,max_damage_structure\n") + for n in range(5): + if (n + 1) % 2 != 0: + dmc = "struct_1" + else: + dmc = "struct_2" + f.write(f"{n+1},area,0,0,{dmc},{dmc},{(n+1)*1000}\n") + + def create_exposure_geoms(): """_summary_.""" geoms = ( @@ -48,9 +74,9 @@ def create_exposure_geoms(): 4.365 52.045, 4.355 52.045))", "POLYGON ((4.395 52.005, 4.395 51.975, 4.415 51.975, \ 4.415 51.985, 4.405 51.985, 4.405 52.005, 4.395 52.005))", - "POLYGON ((4.365 51.960, 4.375 51.990, 4.385 51.960, 4.365 51.960))", - "POLYGON ((4.410 52.030, 4.440 52.030, 4.435 52.010, \ -4.415 52.010, 4.410 52.030))", + "POLYGON ((4.365 51.9605, 4.375 51.9895, 4.385 51.9605, 4.365 51.9605))", + "POLYGON ((4.4105 52.0295, 4.4395 52.0295, 4.435 52.0105, \ +4.415 52.0105, 4.4105 52.0295))", ) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) @@ -276,6 +302,45 @@ def create_hazard_map(): dr = None +def create_hazard_map_highres(): + """_summary_.""" + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + dr = gdal.GetDriverByName("netCDF") + src = dr.Create( + str(Path(p, "hazard", "event_map_highres.nc")), + 100, + 100, + 1, + gdal.GDT_Float32, + ) + gtf = ( + 4.35, + 0.001, + 0.0, + 52.05, + 0.0, + -0.001, + ) + src.SetSpatialRef(srs) + src.SetGeoTransform(gtf) + + band = src.GetRasterBand(1) + data = zeros((100, 100)) + oneD = tuple(range(100)) + for x, y in product(oneD, oneD): + data[x, y] = 3.6 - ((x + y) * 0.02) + band.WriteArray(data) + + band.FlushCache() + src.FlushCache() + + srs = None + band = None + src = None + dr = None + + def create_risk_map(): """_summary_.""" srs = osr.SpatialReference() @@ -396,6 +461,18 @@ def create_settings_geom(): with open(Path(p, "geom_risk_2g.toml"), "wb") as f: tomli_w.dump(doc_r2g, f) + missing_hazard = copy.deepcopy(doc) + del missing_hazard["hazard"]["file"] + + with open(Path(p, "missing_hazard.toml"), "wb") as f: + tomli_w.dump(missing_hazard, f) + + missing_models = copy.deepcopy(doc) + del missing_models["exposure"]["geom"]["file1"] + + with open(Path(p, "missing_models.toml"), "wb") as f: + tomli_w.dump(missing_models, f) + def create_settings_grid(): """_summary_.""" @@ -438,6 +515,12 @@ def create_settings_grid(): with open(Path(p, "grid_risk.toml"), "wb") as f: tomli_w.dump(doc_r, f) + doc_u = copy.deepcopy(doc) + doc_u["hazard"]["file"] = "hazard/event_map_highres.nc" + + with open(Path(p, "grid_unequal.toml"), "wb") as f: + tomli_w.dump(doc_u, f) + def create_vulnerability(): """_summary_.""" @@ -463,11 +546,14 @@ def log_base(b, x): if __name__ == "__main__": create_dbase_stucture() create_exposure_dbase() + create_exposure_dbase_missing() + create_exposure_dbase_partial() create_exposure_geoms() create_exposure_geoms_2() create_exposure_geoms_3() create_exposure_grid() create_hazard_map() + create_hazard_map_highres() create_risk_map() create_settings_geom() create_settings_grid() diff --git a/docs/changelog.qmd b/docs/changelog.qmd index 3100270f..ddb0410f 100644 --- a/docs/changelog.qmd +++ b/docs/changelog.qmd @@ -6,8 +6,13 @@ title: "What's new?" These are the unreleased changes of Delft-FIAT. ### Added +- 'Normal' exit when keyboard interrupt is triggered over cli +- Support for grids with non square cells ### Changed +- Fixed hanging issue with the mp-logging receiver when erroring +- Fixed square cell assumption when mapping world coordinates to pixel coordinates +- Support capitalized entries in the settings toml (again; it was disabled) ### Deprecated diff --git a/pyproject.toml b/pyproject.toml index df082b9e..0bd3e49d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,22 +154,16 @@ ignore_errors = true [tool.coverage.html] directory = ".cov" -## Linting stuff -[tool.black] -line-length = 88 -target-version = ['py311'] - [tool.ruff] line-length = 88 +exclude = ["docs"] -# enable pydocstyle (E), pyflake (F) and isort (I), pytest-style (PT) +[tool.ruff.lint] select = ["E", "F", "I", "PT", "D"] -ignore-init-module-imports = true ignore = ["B904", "D105", "D211", "D213", "D301", "E712", "E741"] -exclude = ["docs"] -[tool.ruff.per-file-ignores] -"test/**" = ["D103", "D100", "D104"] +[tool.ruff.lint.per-file-ignores] +"test/**" = ["D100", "D101", "D102", "D103", "D104"] "test/conftest.py" = ["E402"] "src/fiat/__init__.py" = ["E402", "F401", "F403"] "src/fiat/cli/__init__.py" = ["F403"] @@ -177,5 +171,5 @@ exclude = ["docs"] "src/fiat/methods/__init__.py" = ["F403"] "src/fiat/models/__init__.py" = ["F403"] -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "numpy" diff --git a/src/fiat/cfg.py b/src/fiat/cfg.py index d5015d9c..5ec2e13f 100644 --- a/src/fiat/cfg.py +++ b/src/fiat/cfg.py @@ -12,10 +12,10 @@ check_config_grid, ) from fiat.util import ( - create_hidden_folder, flatten_dict, generic_folder_check, generic_path_check, + get_module_attr, ) @@ -51,10 +51,13 @@ def __init__( f.close() # Initial check for mandatory entries of the settings toml + extra_entries = get_module_attr( + f"fiat.methods.{self.get('global.type', 'flood')}", "MANDATORY_ENTRIES" + ) check_config_entries( self.keys(), self.filepath, - self.path, + extra_entries, ) # Set the cache size per GDAL object @@ -74,9 +77,6 @@ def __init__( self.path, ) self[key] = path - else: - if isinstance(item, str): - self[key] = item.lower() # Switch the build flag off self._build = False @@ -106,16 +106,12 @@ def _create_dir( self, root: Path | str, path: Path | str, - hidden: bool = False, ): """_summary_.""" _p = Path(path) if not _p.is_absolute(): _p = Path(root, _p) - if hidden: - create_hidden_folder(_p) - else: - generic_folder_check(_p) + generic_folder_check(_p) return _p def _create_model_dirs( diff --git a/src/fiat/check.py b/src/fiat/check.py index c3703aae..7d0d7ccf 100644 --- a/src/fiat/check.py +++ b/src/fiat/check.py @@ -16,16 +16,15 @@ def check_config_entries( keys: tuple, path: Path, - parent: Path, + extra_entries: list, ): """_summary_.""" _man_entries = [ "output.path", "hazard.file", "hazard.risk", - "hazard.elevation_reference", "vulnerability.file", - ] + ] + extra_entries _check = [item in keys for item in _man_entries] if not all(_check): @@ -143,7 +142,7 @@ def check_grid_exact( def check_internal_srs( source_srs: osr.SpatialReference, fname: str, - cfg_srs: osr.SpatialReference = None, + cfg_srs: str = None, ): """_summary_.""" if source_srs is None and cfg_srs is None: @@ -245,12 +244,13 @@ def check_hazard_subsets( ## Exposure def check_exp_columns( + index_col: str, columns: tuple | list, specific_columns: tuple | list = [], ): """_summary_.""" _man_columns = [ - "object_id", + index_col, ] + specific_columns _check = [item in columns for item in _man_columns] diff --git a/src/fiat/cli/main.py b/src/fiat/cli/main.py index ce12a164..3c089db1 100644 --- a/src/fiat/cli/main.py +++ b/src/fiat/cli/main.py @@ -28,7 +28,7 @@ ############################################################### Fast Impact Assessment Tool - \u00A9 Deltares + \u00a9 Deltares """ diff --git a/src/fiat/cli/util.py b/src/fiat/cli/util.py index 5cad211b..068c4672 100644 --- a/src/fiat/cli/util.py +++ b/src/fiat/cli/util.py @@ -27,6 +27,9 @@ def run_log( func() except BaseException: t, v, tb = sys.exc_info() - logger.error(",".join([str(item) for item in v.args])) + msg = ",".join([str(item) for item in v.args]) + if t is KeyboardInterrupt: + msg = "KeyboardInterrupt" + logger.error(msg) # Exit with code 1 sys.exit(1) diff --git a/src/fiat/gis/geom.py b/src/fiat/gis/geom.py index dc3fb33a..475bf696 100644 --- a/src/fiat/gis/geom.py +++ b/src/fiat/gis/geom.py @@ -74,7 +74,7 @@ def reproject( if not Path(str(out_dir)).is_dir(): out_dir = gs.path.parent - fname = Path(out_dir, f"{gs.path.stem}_repr_fiat{gs.path.suffix}") + fname = Path(out_dir, f"{gs.path.stem}_repr{gs.path.suffix}") out_srs = osr.SpatialReference() out_srs.SetFromUserInput(crs) diff --git a/src/fiat/gis/grid.py b/src/fiat/gis/grid.py index 70a4d628..39a42f9d 100644 --- a/src/fiat/gis/grid.py +++ b/src/fiat/gis/grid.py @@ -70,8 +70,8 @@ def reproject( if not Path(str(out_dir)).is_dir(): out_dir = gs.path.parent - fname_int = Path(out_dir, f"{gs.path.stem}_repr_fiat.tif") - fname = Path(out_dir, f"{gs.path.stem}_repr_fiat{gs.path.suffix}") + fname_int = Path(out_dir, f"{gs.path.stem}_repr.tif") + fname = Path(out_dir, f"{gs.path.stem}_repr{gs.path.suffix}") out_srs = osr.SpatialReference() out_srs.SetFromUserInput(dst_crs) diff --git a/src/fiat/gis/overlay.py b/src/fiat/gis/overlay.py index 3963799d..3925fb4d 100644 --- a/src/fiat/gis/overlay.py +++ b/src/fiat/gis/overlay.py @@ -85,7 +85,7 @@ def clip( def clip_weighted( - band: gdal.Band, + band: Grid, srs: osr.SpatialReference, gtf: tuple, ft: ogr.Feature, @@ -142,7 +142,7 @@ def clip_weighted( pxWidth = int(lrX - ulX) + 1 pxHeight = int(lrY - ulY) + 1 - clip = band.ReadAsArray(ulX, ulY, pxWidth, pxHeight) + clip = band[ulX, ulY, pxWidth, pxHeight] # m = mask.ReadAsArray(ulX,ulY,pxWidth,pxHeight) # pts = geom.GetGeometryRef(0) @@ -206,8 +206,8 @@ def pin( array A NumPy array containing one value. """ - X, Y = world2pixel(gtf, *point) + x, y = world2pixel(gtf, *point) - value = band[X, Y, 1, 1] + value = band[x, y, 1, 1] return value[0] diff --git a/src/fiat/gis/util.py b/src/fiat/gis/util.py index 5f995eb1..1389c20c 100644 --- a/src/fiat/gis/util.py +++ b/src/fiat/gis/util.py @@ -42,9 +42,10 @@ def world2pixel( ulX = gtf[0] ulY = gtf[3] xDist = gtf[1] - pixel = int((x - ulX) / xDist) - line = int((ulY - y) / xDist) - return (pixel, line) + yDist = gtf[5] + coorX = int((x - ulX) / xDist) + coorY = int((y - ulY) / yDist) + return (coorX, coorY) def pixel2world( diff --git a/src/fiat/io.py b/src/fiat/io.py index f85fea15..25c2f0bd 100644 --- a/src/fiat/io.py +++ b/src/fiat/io.py @@ -581,10 +581,10 @@ def _resolve_column_headers(self): def read( self, - large: bool = False, + lazy: bool = False, ): """_summary_.""" - if large: + if lazy: return TableLazy( data=self.data, index=self.index, @@ -2154,7 +2154,7 @@ def open_csv( delimiter: str = ",", header: bool = True, index: str = None, - large: bool = False, + lazy: bool = False, ) -> object: """Open a csv file. @@ -2168,7 +2168,7 @@ def open_csv( Whether or not to use headers. index : str, optional Name of the index column. - large : bool, optional + lazy : bool, optional If `True`, a lazy read is executed. Returns @@ -2186,7 +2186,7 @@ def open_csv( ) return parser.read( - large=large, + lazy=lazy, ) diff --git a/src/fiat/log.py b/src/fiat/log.py index 9717a5c8..b4e2b5ab 100644 --- a/src/fiat/log.py +++ b/src/fiat/log.py @@ -18,11 +18,13 @@ DEFAULT_FMT = "{asctime:20s}{levelname:8s}{message}" DEFAULT_TIME_FMT = "%Y-%m-%d %H:%M:%S" +RECEIVER_COUNT = 1 STREAM_COUNT = 1 _Global_and_Destruct_Lock = threading.RLock() _handlers = weakref.WeakValueDictionary() _loggers = weakref.WeakValueDictionary() +_receivers = weakref.WeakValueDictionary() _str_formatter = StrFormatter() del StrFormatter @@ -49,6 +51,9 @@ def _Destruction(): handler.flush() handler.close() handler.release() + items = list(_receivers.items()) + for _, receiver in items: + receiver.close() atexit.register(_Destruction) @@ -580,11 +585,26 @@ def __init__( self, queue: object, ): + self._closed = False self._t = None self._handlers = [] self.count = 0 self.q = queue + global RECEIVER_COUNT + self._name = f"Receiver{RECEIVER_COUNT}" + RECEIVER_COUNT += 1 + + self._add_global_receiver_ref() + + def _add_global_receiver_ref( + self, + ): + """_summary_.""" + global_acquire() + _receivers[self._name] = self + global_release() + def _log( self, record: LogItem, @@ -607,9 +627,14 @@ def _waiting(self): def close(self): """Close the receiver.""" - self.q.put_nowait(self._sentinel) - self._t.join() - self._t = None + if not self._closed: + self.q.put_nowait(self._sentinel) + self._t.join() + self._t = None + self._closed = True + global_acquire() + del _receivers[self._name] + global_release() def close_handlers(self): """Close all associated handlers.""" @@ -651,8 +676,8 @@ def start(self): self._t = t = threading.Thread( target=self._waiting, name="mp_logging_thread", + daemon=True, ) - t.deamon = True t.start() diff --git a/src/fiat/methods/ead.py b/src/fiat/methods/ead.py index 9a1b5871..e2461ee2 100644 --- a/src/fiat/methods/ead.py +++ b/src/fiat/methods/ead.py @@ -1,4 +1,5 @@ """EAD (Expected Annual Damages) related functionality.""" + import math diff --git a/src/fiat/methods/flood.py b/src/fiat/methods/flood.py index 5812cac2..03d3f59c 100644 --- a/src/fiat/methods/flood.py +++ b/src/fiat/methods/flood.py @@ -1,4 +1,5 @@ """Functions specifically for flood risk calculation.""" + import math from numpy import isnan @@ -8,7 +9,7 @@ from fiat.methods.util import AREA_METHODS MANDATORY_COLUMNS = ["ground_flht", "ground_elevtn"] -MANDATORY_ENTRIES = {"ref": "hazard.elevation_reference"} +MANDATORY_ENTRIES = ["hazard.elevation_reference"] NEW_COLUMNS = ["inun_depth"] diff --git a/src/fiat/models/geom.py b/src/fiat/models/geom.py index 28f18e1c..b848cf0f 100644 --- a/src/fiat/models/geom.py +++ b/src/fiat/models/geom.py @@ -81,6 +81,7 @@ def _discover_exposure_meta( columns: dict, meta: dict, index: int, + index_col: str, ): """Simple method for sorting out the exposure meta.""" # noqa: D401 # check if set from the csv file @@ -88,7 +89,8 @@ def _discover_exposure_meta( meta[index] = {} # Check the exposure column headers check_exp_columns( - list(columns.keys()), + index_col, + columns=list(columns.keys()), specific_columns=getattr(self.module, "MANDATORY_COLUMNS"), ) @@ -181,10 +183,16 @@ def get_exposure_meta(self): self.exposure_data._columns, meta, -1, + self.cfg.get("exposure.csv.settings.index"), ) for key, gm in self.exposure_geoms.items(): columns = gm._columns - self._discover_exposure_meta(columns, meta, key) + self._discover_exposure_meta( + columns, + meta, + key, + self.cfg.get("exposure.geom.settings.index"), + ) self.cfg.set("_exposure_meta", meta) def read_exposure(self): @@ -204,7 +212,8 @@ def read_exposure_data(self): kw.update( self.cfg.generate_kwargs("exposure.csv.settings"), ) - data = open_csv(path, large=True, **kw) + self.cfg.set("exposure.csv.settings.index", kw["index"]) + data = open_csv(path, lazy=True, **kw) ##checks logger.info("Executing exposure data checks...") diff --git a/src/fiat/models/worker_geom.py b/src/fiat/models/worker_geom.py index 8f427206..a5a2423c 100644 --- a/src/fiat/models/worker_geom.py +++ b/src/fiat/models/worker_geom.py @@ -39,7 +39,7 @@ def worker( func_hazard = getattr(module, "calculate_hazard") func_damage = getattr(module, "calculate_damage") man_columns = getattr(module, "MANDATORY_COLUMNS") - man_entries = tuple(getattr(module, "MANDATORY_ENTRIES").values()) + man_entries = getattr(module, "MANDATORY_ENTRIES") # Get the bands to prevent object creation while looping bands = [(haz[idx + 1], idx + 1) for idx in range(haz.size)] diff --git a/src/fiat/models/worker_grid.py b/src/fiat/models/worker_grid.py index de817102..52a45728 100644 --- a/src/fiat/models/worker_grid.py +++ b/src/fiat/models/worker_grid.py @@ -1,4 +1,5 @@ """Worker functions for grid model.""" + from math import floor from pathlib import Path diff --git a/src/fiat/util.py b/src/fiat/util.py index 236ca56e..f2d2f5d9 100644 --- a/src/fiat/util.py +++ b/src/fiat/util.py @@ -1,10 +1,9 @@ """Base FIAT utility.""" -import ctypes import fnmatch +import importlib import math import os -import platform import re import sys from collections.abc import MutableMapping @@ -356,31 +355,6 @@ def generic_folder_check( path.mkdir(parents=True) -def create_hidden_folder( - path: Path | str, -): - """_summary_. - - Parameters - ---------- - path : Path | str - _description_ - """ - path = Path(path) - if not path.stem.startswith("."): - path = Path(path.parent, f".{path.stem}") - generic_folder_check(path) - - if platform.system().lower() == "windows": - r = ctypes.windll.kernel32.SetFileAttributesW( - str(path), - FILE_ATTRIBUTE_HIDDEN, - ) - - if not r: - raise OSError("") - - def generic_path_check( path: str, root: str, @@ -454,6 +428,14 @@ def find_duplicates(elements: tuple | list): return dup +def get_module_attr(module: str, attr: str): + """Quickly get attribute from a module dynamically.""" + module = importlib.import_module(module) + out = getattr(module, attr) + module = None + return out + + def object_size(obj): """Calculate the actual size of an object (bit overestimated). diff --git a/test/conftest.py b/test/conftest.py index c10fcceb..7e5d2888 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest + from fiat.cfg import ConfigReader from fiat.cli.main import args_parser from fiat.io import open_csv, open_geom, open_grid @@ -15,77 +16,98 @@ "geom_risk_2g", "grid_event", "grid_risk", + "grid_unequal", + "missing_hazard", + "missing_models", ] +_PATH = Path.cwd() -@pytest.fixture() +@pytest.fixture def cli_parser(): return args_parser() -@pytest.fixture() +@pytest.fixture def settings_files(): _files = {} for _m in _MODELS: - p = Path(Path.cwd(), ".testdata", f"{_m}.toml") + p = Path(_PATH, ".testdata", f"{_m}.toml") p_name = p.stem _files[p_name] = p return _files -@pytest.fixture() +@pytest.fixture def configs(settings_files): _cfgs = {} for key, item in settings_files.items(): - _cfgs[key] = ConfigReader(item) + if not key.startswith("missing"): + _cfgs[key] = ConfigReader(item) return _cfgs -@pytest.fixture() +@pytest.fixture def geom_risk(configs): model = GeomModel(configs["geom_risk"]) return model -@pytest.fixture() +@pytest.fixture def grid_risk(configs): model = GridModel(configs["grid_risk"]) return model -@pytest.fixture() +@pytest.fixture def geom_data(): - d = open_geom(Path(Path.cwd(), ".testdata", "exposure", "spatial.gpkg")) + d = open_geom(Path(_PATH, ".testdata", "exposure", "spatial.gpkg")) return d -@pytest.fixture() +@pytest.fixture +def geom_partial_data(): + d = open_csv(Path(_PATH, ".testdata", "exposure", "spatial_partial.csv"), lazy=True) + return d + + +@pytest.fixture def grid_event_data(): - d = open_grid(Path(Path.cwd(), ".testdata", "hazard", "event_map.nc")) + d = open_grid(Path(_PATH, ".testdata", "hazard", "event_map.nc")) + return d + + +@pytest.fixture +def grid_event_highres_data(): + d = open_grid(Path(_PATH, ".testdata", "hazard", "event_map_highres.nc")) + return d + + +@pytest.fixture +def grid_exp_data(): + d = open_grid(Path(_PATH, ".testdata", "exposure", "spatial.nc")) return d -@pytest.fixture() +@pytest.fixture def grid_risk_data(): - d = open_grid(Path(Path.cwd(), ".testdata", "hazard", "risk_map.nc")) + d = open_grid(Path(_PATH, ".testdata", "hazard", "risk_map.nc")) return d -@pytest.fixture() +@pytest.fixture def vul_data(): - d = open_csv( - Path(Path.cwd(), ".testdata", "vulnerability", "vulnerability_curves.csv") - ) + d = open_csv(Path(_PATH, ".testdata", "vulnerability", "vulnerability_curves.csv")) return d -@pytest.fixture() +@pytest.fixture def log1(): obj = LogItem(level=2, msg="Hello!") return obj -@pytest.fixture() +@pytest.fixture def log2(): obj = LogItem(level=2, msg="Good Bye!") return obj diff --git a/test/test_checks.py b/test/test_checks.py new file mode 100644 index 00000000..6cf5f68b --- /dev/null +++ b/test/test_checks.py @@ -0,0 +1,153 @@ +import sys +from pathlib import Path + +from osgeo import osr + +from fiat import ConfigReader, GeomModel, GridModel +from fiat.check import ( + check_exp_derived_types, + check_hazard_rp, + check_hazard_subsets, + check_internal_srs, +) +from fiat.error import FIATDataError +from fiat.util import discover_exp_columns + + +def test_check_config_entries(settings_files): + settings = settings_files["missing_hazard"] + + try: + _ = ConfigReader(settings) + except FIATDataError: + t, v, tb = sys.exc_info() + assert v.msg.startswith("Missing mandatory entries") + assert v.msg.endswith("['hazard.file']") + finally: + assert v + + +def test_check_config_models(settings_files): + neither = settings_files["missing_models"] + cfg = ConfigReader(neither) + assert cfg.get_model_type() == [False, False] + + geom = settings_files["geom_event"] + cfg = ConfigReader(geom) + assert cfg.get_model_type() == [True, False] + + geom = settings_files["grid_event"] + cfg = ConfigReader(geom) + assert cfg.get_model_type() == [False, True] + + +def test_check_exp_columns(configs): + cfg = configs["geom_event"] + cfg.set( + "exposure.csv.file", + Path(Path.cwd(), ".testdata", "exposure", "spatial_missing.csv"), + ) + + try: + _ = GeomModel(cfg) + except FIATDataError: + t, v, tb = sys.exc_info() + assert v.msg == "Missing mandatory exposure columns: ['object_id']" + finally: + assert v + + +def test_check_exp_derived_types(geom_partial_data): + found, found_idx, missing = discover_exp_columns( + geom_partial_data._columns, type="damage" + ) + assert missing == ["content"] + check_exp_derived_types("damage", found, missing) + + found = [] + try: + check_exp_derived_types("damage", found, missing) + except FIATDataError: + t, v, tb = sys.exc_info() + assert v.msg.startswith("For type: 'damage' no matching") + finally: + assert v + + +def test_check_exp_index_col(configs): + cfg = configs["geom_event"] + cfg.set("exposure.geom.settings.index", "faulty") + + try: + _ = GeomModel(cfg) + except FIATDataError: + t, v, tb = sys.exc_info() + assert v.msg.startswith("Index column ('faulty') not found") + finally: + assert v + + +def test_check_grid_exact(configs): + exact = configs["grid_event"] + model = GridModel(exact) + assert model.equal == True + + unequal = configs["grid_unequal"] + model = GridModel(unequal) + assert model.equal == False + assert model.cfg.get("hazard.file").exists() + + +def test_check_hazard_rp(): + rp_bands = ["a", "b", "c", "d"] + rp_cfg = [1, 2, 5, 10] + + out = check_hazard_rp(rp_bands, rp_cfg, "") + assert out == [1.0, 2.0, 5.0, 10.0] + + rp_cfg.remove(10) + try: + _ = check_hazard_rp(rp_bands, rp_cfg, Path("file.ext")) + except FIATDataError: + t, v, tb = sys.exc_info() + assert v.msg.startswith( + "'file.ext': cannot determine the return periods \ +for the risk calculation" + ) + finally: + assert v + + +def test_check_hazard_subsets(grid_event_data, grid_risk_data): + assert grid_event_data.subset_dict is None + check_hazard_subsets(grid_event_data.subset_dict, "") + + try: + assert grid_risk_data.subset_dict is not None + check_hazard_subsets(grid_risk_data.subset_dict, Path("file.ext")) + except FIATDataError: + t, v, tb = sys.exc_info() + assert v.msg.startswith("'file.ext': cannot read this file as there") + assert v + + +def test_check_internal_srs(): + try: + check_internal_srs(None, "file", None) + except FIATDataError: + t, v, tb = sys.exc_info() + assert v.msg.startswith("Coordinate reference system is unknown for 'file'") + finally: + assert v + + s = osr.SpatialReference() + s.ImportFromEPSG(4326) + + int_srs = check_internal_srs(s, fname="") + assert int_srs is None + s = None + + int_srs = check_internal_srs(None, "", "EPSG:4326") + assert int_srs is not None + assert int_srs.GetAuthorityCode(None) == "4326" + int_srs = None diff --git a/test/test_gis.py b/test/test_gis.py index 6b44ce67..8a9908bf 100644 --- a/test/test_gis.py +++ b/test/test_gis.py @@ -1,18 +1,51 @@ -from fiat.gis import geom, grid, overlay from numpy import mean +from fiat.gis import geom, grid, overlay +from fiat.gis.crs import get_srs_repr -def test_clip_grid_geom(geom_data, grid_event_data): - for ft in geom_data: - hazard = overlay.clip( - grid_event_data[1], - grid_event_data.get_srs(), - grid_event_data.get_geotransform(), - ft, - ) - assert len(hazard) == 7 - assert int(round(mean(hazard) * 100, 0)) == 166 +def test_clip(geom_data, grid_event_data): + ft = geom_data[4] + hazard = overlay.clip( + grid_event_data[1], + grid_event_data.get_srs(), + grid_event_data.get_geotransform(), + ft, + ) + ft = None + + assert len(hazard) == 6 + assert int(round(mean(hazard) * 100, 0)) == 170 + + +def test_clip_weighted(geom_data, grid_event_data): + ft = geom_data[4] + _, weights = overlay.clip_weighted( + grid_event_data[1], + grid_event_data.get_srs(), + grid_event_data.get_geotransform(), + ft, + upscale=10, + ) + assert int(weights[0, 0] * 100) == 90 + + _, weights = overlay.clip_weighted( + grid_event_data[1], + grid_event_data.get_srs(), + grid_event_data.get_geotransform(), + ft, + upscale=100, + ) + assert int(weights[0, 0] * 100) == 80 + + _, weights = overlay.clip_weighted( + grid_event_data[1], + grid_event_data.get_srs(), + grid_event_data.get_geotransform(), + ft, + upscale=1000, + ) + assert int(weights[0, 0] * 100) == 79 def test_pin(geom_data, grid_event_data): @@ -28,20 +61,37 @@ def test_pin(geom_data, grid_event_data): assert int(round(hazard[0] * 100, 0)) == 160 -def test_reproject(tmp_path, geom_data, grid_event_data): +def test_geom_reproject(tmp_path, geom_data, grid_event_data): dst_crs = "EPSG:3857" - new_gm = geom.reproject( geom_data, dst_crs, out_dir=str(tmp_path), ) + assert new_gm.get_srs().GetAuthorityCode(None) == "3857" + + +def test_grid_reproject(tmp_path, grid_event_data): + dst_crs = "EPSG:3857" new_gr = grid.reproject( grid_event_data, dst_crs, out_dir=str(tmp_path), ) - assert new_gm.get_srs().GetAuthorityCode(None) == "3857" assert new_gr.get_srs().GetAuthorityCode(None) == "3857" + + +def test_grid_reproject_gtf(tmp_path, grid_event_data, grid_event_highres_data): + assert grid_event_highres_data.shape == (100, 100) + new_gr = grid.reproject( + grid_event_highres_data, + get_srs_repr(grid_event_data.get_srs()), + dst_gtf=grid_event_data.get_geotransform(), + dst_width=10, + dst_height=10, + out_dir=str(tmp_path), + ) + + assert new_gr.shape == (10, 10) diff --git a/test/test_model.py b/test/test_model.py index 2a9b1eae..66690daf 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -1,8 +1,9 @@ from pathlib import Path +from osgeo import gdal + from fiat import FIAT from fiat.io import open_csv -from osgeo import gdal def run_model(cfg, p): diff --git a/test/test_struct.py b/test/test_struct.py index 990d79d4..62a13fee 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -9,7 +9,7 @@ def test_geomsource(geom_data): bounds = geom_data.bounds bounds = [round(item * 10000) for item in bounds] - assert bounds == [43550, 44400, 519600, 520450] + assert bounds == [43550, 44395, 519605, 520450] assert geom_data.fields == ["object_id", "object_name"]