From e4c56f0b16c2d6488ea45dabdf3ec6cf771d62ae Mon Sep 17 00:00:00 2001 From: Sam Vente Date: Wed, 6 Sep 2023 12:48:51 +0200 Subject: [PATCH 1/6] add tables property to base model class --- .gitignore | 1 + hydromt/models/model_api.py | 128 ++++++++++++++++++++++++++++++++---- tests/test_model.py | 41 ++++++++++++ 3 files changed, 159 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 42c4a83d1..c2d72c2db 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,4 @@ dask-worker-space/ #ruff linting .ruff_cache .envrc +pyrightconfig.json diff --git a/hydromt/models/model_api.py b/hydromt/models/model_api.py index 0256f4b8e..2291b1314 100644 --- a/hydromt/models/model_api.py +++ b/hydromt/models/model_api.py @@ -12,7 +12,7 @@ from os.path import abspath, basename, dirname, isabs, isdir, isfile, join from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union +from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, TypedDict, Union import geopandas as gpd import numpy as np @@ -35,6 +35,7 @@ "DeferedFileClose", {"ds": xr.Dataset, "org_fn": str, "tmp_fn": str, "close_attempts": int}, ) +XArrayDict = Dict[str, Union[xr.DataArray, xr.Dataset]] class Model(object, metaclass=ABCMeta): @@ -57,11 +58,11 @@ class Model(object, metaclass=ABCMeta): "crs": CRS, "config": Dict[str, Any], "geoms": Dict[str, gpd.GeoDataFrame], - "maps": Dict[str, Union[xr.DataArray, xr.Dataset]], - "forcing": Dict[str, Union[xr.DataArray, xr.Dataset]], + "maps": XArrayDict, + "forcing": XArrayDict, "region": gpd.GeoDataFrame, - "results": Dict[str, Union[xr.DataArray, xr.Dataset]], - "states": Dict[str, Union[xr.DataArray, xr.Dataset]], + "results": XArrayDict, + "states": XArrayDict, } def __init__( @@ -108,13 +109,14 @@ def __init__( # metadata maps that can be at different resolutions # TODO do we want read/write maps? self._config = None # nested dictionary - self._maps = None # dictionary of xr.DataArray and/or xr.Dataset + self._maps: Optional[XArrayDict] = None + self._tables: Dict[str, pd.Dataframe] = {} # NOTE was staticgeoms in <=v0.5 - self._geoms = None # dictionary of gdp.GeoDataFrame - self._forcing = None # dictionary of xr.DataArray and/or xr.Dataset - self._states = None # dictionary of xr.DataArray and/or xr.Dataset - self._results = None # dictionary of xr.DataArray and/or xr.Dataset + self._geoms: Optional[Dict[str, gpd.GeoDataFrame]] = None + self._forcing: Optional[XArrayDict] = None + self._states: Optional[XArrayDict] = None + self._results: Optional[XArrayDict] = None # To be deprecated in future versions! self._staticmaps = None self._staticgeoms = None @@ -702,6 +704,110 @@ def _configread(self, fn: str): def _configwrite(self, fn: str): return config.configwrite(fn, self.config) + def write_tables( + self, driver: Literal["csv", "parquet", "excel"], **kwargs + ) -> None: + """Write tables at .""" + if not self._write: + raise IOError("Model opened in read-only mode") + + if self._tables: + self.logger.info("Writing table files.") + for name in self._tables: + if driver == "csv": + fn_out = join(self.root, f"{name}.csv") + local_kwargs = {"index": False, "header": True, "sep": ","} + local_kwargs.update(**kwargs) + self._tables[name].to_csv(fn_out, **local_kwargs) + elif driver == "parquet": + fn_out = join(self.root, f"{name}.parquet") + local_kwargs = kwargs.copy() + self._tables[name].to_parquet(fn_out, **local_kwargs) + elif driver == "excel": + fn_out = join(self.root, f"{name}.xlsx") + local_kwargs = kwargs.copy() + local_kwargs = {"index": False} + self._tables[name].to_excel(fn_out, **local_kwargs) + else: + raise ValueError( + f"Driver {driver} not recognised. " + "options are: ['csv','parquet','excel']" + ) + + def read_tables(self, driver: Literal["csv", "parquet", "excel"], **kwargs) -> None: + """Read table files at and parse to dict of dataframes.""" + if not self._write: + self._tables = dict() # start fresh in read-only mode + + if driver == "csv": + ext = "csv" + driver_fn = pd.read_csv + elif driver == "parquet": + ext = "parquet" + driver_fn = pd.read_parquet + elif driver == "excel": + ext = "xlsx" + driver_fn = pd.read_excel + + else: + raise ValueError( + f"Driver {driver} not recognised. " + "options are: ['csv','parquet','excel']" + ) + + self.logger.info("Reading model table files.") + fns = glob.glob(join(self.root, f"*.{ext}")) + if len(fns) > 0: + for fn in fns: + name = basename(fn).split(".")[0] + tbl = driver_fn(fn, **kwargs) + self.set_table(name, tbl) + else: + self.logger.warning( + f"Attempted to read tables with extention {ext}," + " but no files were found in root." + ) + + def get_table(self, name: str) -> pd.DataFrame: + """Return the table associated with a given name.""" + return self._tables[name] + + def set_table(self, name, df) -> None: + """Add a table to model.""" + if not (isinstance(df, pd.DataFrame) or isinstance(df, pd.Series)): + raise ValueError( + "df type not recognized, should be pandas Dataframe or Series." + ) + if name in self._tables: + if not self._write: + raise IOError(f"Cannot overwrite table {name} in read-only mode") + elif self._read: + self.logger.warning(f"Overwriting table: {name}") + + self._tables[name] = df + + def set_tables(self, table_dict) -> None: + """Add a table to model.""" + for name, df in table_dict.items(): + self.set_table(name, df) + + def get_tables_merged(self) -> pd.DataFrame: + """Return all tables of a model merged into one dataframe. + + This is mostly used for convinience and testing. + """ + return pd.concat( + [df.assign(table_origin=name) for name, df in self._tables.items()], axis=0 + ) + + def get_table_names(self) -> Iterable[str]: + """Return the names of all known tables.""" + return self._tables.keys() + + def iter_tables(self) -> Iterable[Tuple[str, pd.DataFrame]]: + """Iterate over all known tables and their names.""" + return iter(self._tables.items()) + def read_config(self, config_fn: Optional[str] = None): """Parse config from file. @@ -1481,7 +1587,7 @@ def _cleanup(self, forceful_overwrite=False, max_close_attempts=2) -> List[str]: def write_nc( self, - nc_dict: Dict[str, Union[xr.DataArray, xr.Dataset]], + nc_dict: XArrayDict, fn: str, gdal_compliant: bool = False, rename_dims: bool = False, diff --git a/tests/test_model.py b/tests/test_model.py index 963317735..19bcd71dc 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Tests for the hydromt.models module of HydroMT.""" +from copy import deepcopy from os.path import abspath, dirname, isfile, join import geopandas as gpd @@ -171,6 +172,46 @@ def test_model(model, tmpdir): assert np.all(model.region.total_bounds == model.staticmaps.raster.bounds) +@pytest.mark.parametrize("driver", ["csv", "excel", "parquet"]) +def test_model_read_write_tables(model, df, driver, tmpdir): + model.set_root(tmpdir) + # make a couple copies of the dfs for testing + dfs = {str(i): df.copy() for i in range(5)} + clean_model = deepcopy(model) + model.set_tables(dfs) + + model.write_tables(driver=driver) + clean_model.read_tables(driver=driver) + + model_merged = model.get_tables_merged().sort_values(["table_origin", "city"]) + clean_model_merged = clean_model.get_tables_merged().sort_values( + ["table_origin", "city"] + ) + assert np.all( + np.equal(model_merged, clean_model_merged) + ), f"model: {model_merged}\nclean_model: {clean_model_merged}" + + +def test_model_tables(model, df, tmpdir): + # make a couple copies of the dfs for testing + dfs = {str(i): df.copy() for i in range(5)} + model.set_root(tmpdir) + + with pytest.raises(KeyError): + model.get_table(1) + + for i, d in dfs.items(): + model.set_table(i, d) + assert df.equals(model.get_table(i)) + + # now do the same but interating over the stables instead + for i, d in model.iter_tables(): + model.set_table(i, d) + assert df.equals(model.get_table(i)) + + assert list(model.get_table_names()) == list(map(str, range(5))) + + def test_model_append(demda, tmpdir): # write a model demda.name = "dem" From 099cfb3b4e9404ab1f1b283ee2ce57a0d9b6e59f Mon Sep 17 00:00:00 2001 From: Sam Vente Date: Wed, 6 Sep 2023 12:52:31 +0200 Subject: [PATCH 2/6] update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e9212c611..eca0c3b2d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,7 @@ Added - new ``force-overwrite`` option in ``hydromt update`` CLI to force overwritting updated netcdf files. (PR #460) - add ``open_mfcsv`` function in ``io`` module for combining multiple CSV files into one dataset. (PR #486) - Adapters can now clip data that is passed through a python object the same way as through the data catalog. (PR #481) +- Model class now has methods for getting, setting, reading and writing arbitrary tabular data. (PR #502) Changed ------- From 57b8a38373534c9956112758547ab5eaf9adb6d4 Mon Sep 17 00:00:00 2001 From: Sam Vente Date: Thu, 7 Sep 2023 14:50:18 +0200 Subject: [PATCH 3/6] use dynamic interface like other properties --- hydromt/models/model_api.py | 93 +++++++++++-------------------------- tests/test_model.py | 42 +++++++---------- 2 files changed, 43 insertions(+), 92 deletions(-) diff --git a/hydromt/models/model_api.py b/hydromt/models/model_api.py index 2291b1314..9378ece2f 100644 --- a/hydromt/models/model_api.py +++ b/hydromt/models/model_api.py @@ -12,7 +12,7 @@ from os.path import abspath, basename, dirname, isabs, isdir, isfile, join from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, TypedDict, Union +from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union import geopandas as gpd import numpy as np @@ -110,7 +110,7 @@ def __init__( # TODO do we want read/write maps? self._config = None # nested dictionary self._maps: Optional[XArrayDict] = None - self._tables: Dict[str, pd.Dataframe] = {} + self._tables: Dict[str, pd.DataFrame] = None # NOTE was staticgeoms in <=v0.5 self._geoms: Optional[Dict[str, gpd.GeoDataFrame]] = None @@ -704,87 +704,54 @@ def _configread(self, fn: str): def _configwrite(self, fn: str): return config.configwrite(fn, self.config) - def write_tables( - self, driver: Literal["csv", "parquet", "excel"], **kwargs - ) -> None: + @property + def tables(self) -> Dict[str, pd.DataFrame]: + """Model tables.""" + if self._tables is None: + self._tables = dict() + if self._read: + self.read_tables() + return self._tables + + def write_tables(self, **kwargs) -> None: """Write tables at .""" if not self._write: raise IOError("Model opened in read-only mode") - if self._tables: + if self.tables: self.logger.info("Writing table files.") - for name in self._tables: - if driver == "csv": - fn_out = join(self.root, f"{name}.csv") - local_kwargs = {"index": False, "header": True, "sep": ","} - local_kwargs.update(**kwargs) - self._tables[name].to_csv(fn_out, **local_kwargs) - elif driver == "parquet": - fn_out = join(self.root, f"{name}.parquet") - local_kwargs = kwargs.copy() - self._tables[name].to_parquet(fn_out, **local_kwargs) - elif driver == "excel": - fn_out = join(self.root, f"{name}.xlsx") - local_kwargs = kwargs.copy() - local_kwargs = {"index": False} - self._tables[name].to_excel(fn_out, **local_kwargs) - else: - raise ValueError( - f"Driver {driver} not recognised. " - "options are: ['csv','parquet','excel']" - ) + local_kwargs = {"index": False, "header": True, "sep": ","} + local_kwargs.update(**kwargs) + for name in self.tables: + fn_out = join(self.root, f"{name}.csv") + self.tables[name].to_csv(fn_out, **local_kwargs) - def read_tables(self, driver: Literal["csv", "parquet", "excel"], **kwargs) -> None: + def read_tables(self, **kwargs) -> None: """Read table files at and parse to dict of dataframes.""" if not self._write: - self._tables = dict() # start fresh in read-only mode - - if driver == "csv": - ext = "csv" - driver_fn = pd.read_csv - elif driver == "parquet": - ext = "parquet" - driver_fn = pd.read_parquet - elif driver == "excel": - ext = "xlsx" - driver_fn = pd.read_excel - - else: - raise ValueError( - f"Driver {driver} not recognised. " - "options are: ['csv','parquet','excel']" - ) + self.tables = dict() # start fresh in read-only mode self.logger.info("Reading model table files.") - fns = glob.glob(join(self.root, f"*.{ext}")) + fns = glob.glob(join(self.root, "*.csv")) if len(fns) > 0: for fn in fns: name = basename(fn).split(".")[0] - tbl = driver_fn(fn, **kwargs) + tbl = pd.read_csv(fn, **kwargs) self.set_table(name, tbl) - else: - self.logger.warning( - f"Attempted to read tables with extention {ext}," - " but no files were found in root." - ) - - def get_table(self, name: str) -> pd.DataFrame: - """Return the table associated with a given name.""" - return self._tables[name] def set_table(self, name, df) -> None: """Add a table to model.""" if not (isinstance(df, pd.DataFrame) or isinstance(df, pd.Series)): raise ValueError( - "df type not recognized, should be pandas Dataframe or Series." + "df type not recognized, should be pandas DataFrame or Series." ) - if name in self._tables: + if name in self.tables: if not self._write: raise IOError(f"Cannot overwrite table {name} in read-only mode") elif self._read: self.logger.warning(f"Overwriting table: {name}") - self._tables[name] = df + self.tables[name] = df def set_tables(self, table_dict) -> None: """Add a table to model.""" @@ -797,17 +764,9 @@ def get_tables_merged(self) -> pd.DataFrame: This is mostly used for convinience and testing. """ return pd.concat( - [df.assign(table_origin=name) for name, df in self._tables.items()], axis=0 + [df.assign(table_origin=name) for name, df in self.tables.items()], axis=0 ) - def get_table_names(self) -> Iterable[str]: - """Return the names of all known tables.""" - return self._tables.keys() - - def iter_tables(self) -> Iterable[Tuple[str, pd.DataFrame]]: - """Iterate over all known tables and their names.""" - return iter(self._tables.items()) - def read_config(self, config_fn: Optional[str] = None): """Parse config from file. diff --git a/tests/test_model.py b/tests/test_model.py index 19bcd71dc..cde3bf82a 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -172,44 +172,36 @@ def test_model(model, tmpdir): assert np.all(model.region.total_bounds == model.staticmaps.raster.bounds) -@pytest.mark.parametrize("driver", ["csv", "excel", "parquet"]) -def test_model_read_write_tables(model, df, driver, tmpdir): - model.set_root(tmpdir) - # make a couple copies of the dfs for testing - dfs = {str(i): df.copy() for i in range(5)} - clean_model = deepcopy(model) - model.set_tables(dfs) - - model.write_tables(driver=driver) - clean_model.read_tables(driver=driver) - - model_merged = model.get_tables_merged().sort_values(["table_origin", "city"]) - clean_model_merged = clean_model.get_tables_merged().sort_values( - ["table_origin", "city"] - ) - assert np.all( - np.equal(model_merged, clean_model_merged) - ), f"model: {model_merged}\nclean_model: {clean_model_merged}" - - def test_model_tables(model, df, tmpdir): # make a couple copies of the dfs for testing dfs = {str(i): df.copy() for i in range(5)} model.set_root(tmpdir) + clean_model = deepcopy(model) with pytest.raises(KeyError): - model.get_table(1) + model.tables[1] for i, d in dfs.items(): model.set_table(i, d) - assert df.equals(model.get_table(i)) + assert df.equals(model.tables[i]) # now do the same but interating over the stables instead - for i, d in model.iter_tables(): + for i, d in model.tables.items(): model.set_table(i, d) - assert df.equals(model.get_table(i)) + assert df.equals(model.tables[i]) - assert list(model.get_table_names()) == list(map(str, range(5))) + assert list(model.tables.keys()) == list(map(str, range(5))) + + model.write_tables() + clean_model.read_tables() + + model_merged = model.get_tables_merged().sort_values(["table_origin", "city"]) + clean_model_merged = clean_model.get_tables_merged().sort_values( + ["table_origin", "city"] + ) + assert np.all( + np.equal(model_merged, clean_model_merged) + ), f"model: {model_merged}\nclean_model: {clean_model_merged}" def test_model_append(demda, tmpdir): From 5e52b5112c07690bcfecb1e8bd06c95c631e2197 Mon Sep 17 00:00:00 2001 From: Dirk Eilander Date: Thu, 7 Sep 2023 15:44:28 +0200 Subject: [PATCH 4/6] fix read/write; improve consistency between properties --- hydromt/models/model_api.py | 76 +++++++++++++++++++-------------- hydromt/models/model_grid.py | 17 ++++++-- hydromt/models/model_lumped.py | 6 ++- hydromt/models/model_mesh.py | 5 ++- hydromt/models/model_network.py | 6 ++- tests/test_model.py | 9 ++-- 6 files changed, 73 insertions(+), 46 deletions(-) diff --git a/hydromt/models/model_api.py b/hydromt/models/model_api.py index 9caddabd0..007a082d2 100644 --- a/hydromt/models/model_api.py +++ b/hydromt/models/model_api.py @@ -511,6 +511,7 @@ def read( "config", "staticmaps", "maps", + "tables", "geoms", "forcing", "states", @@ -539,6 +540,7 @@ def write( components: List = [ "staticmaps", "maps", + "tables", "geoms", "forcing", "states", @@ -716,56 +718,64 @@ def tables(self) -> Dict[str, pd.DataFrame]: self.read_tables() return self._tables - def write_tables(self, **kwargs) -> None: - """Write tables at .""" - if not self._write: - raise IOError("Model opened in read-only mode") - + def write_tables(self, fn: str = "tables/{name}.csv", **kwargs) -> None: + """Write tables at /tables.""" if self.tables: + self._assert_write_mode self.logger.info("Writing table files.") local_kwargs = {"index": False, "header": True, "sep": ","} local_kwargs.update(**kwargs) for name in self.tables: - fn_out = join(self.root, f"{name}.csv") + fn_out = join(self.root, fn.format(name=name)) + os.makedirs(dirname(fn_out), exist_ok=True) self.tables[name].to_csv(fn_out, **local_kwargs) + else: + self.logger.debug("No tables found, skip writing.") - def read_tables(self, **kwargs) -> None: - """Read table files at and parse to dict of dataframes.""" - if not self._write: - self.tables = dict() # start fresh in read-only mode - + def read_tables(self, fn: str = "tables/{name}.csv", **kwargs) -> None: + """Read table files at /tables and parse to dict of dataframes.""" + self._assert_read_mode self.logger.info("Reading model table files.") - fns = glob.glob(join(self.root, "*.csv")) + fns = glob.glob(join(self.root, fn.format(name="*"))) if len(fns) > 0: for fn in fns: name = basename(fn).split(".")[0] tbl = pd.read_csv(fn, **kwargs) - self.set_table(name, tbl) + self.set_tables(tbl, name=name) - def set_table(self, name, df) -> None: - """Add a table to model.""" - if not (isinstance(df, pd.DataFrame) or isinstance(df, pd.Series)): - raise ValueError( - "df type not recognized, should be pandas DataFrame or Series." - ) - if name in self.tables: - if not self._write: - raise IOError(f"Cannot overwrite table {name} in read-only mode") - elif self._read: - self.logger.warning(f"Overwriting table: {name}") + def set_tables( + self, tables: Union[pd.DataFrame, pd.Series, Dict], name=None + ) -> None: + """Add (a) table(s) to model. - self.tables[name] = df + Parameters + ---------- + tables : pandas.DataFrame, pandas.Series or dict + Table(s) to add to model. + Multiple tables can be added at once by passing a dict of tables. + name : str, optional + Name of table, by default None. Required when tables is not a dict. + """ + if not isinstance(tables, dict) and name is None: + raise ValueError("name required when tables is not a dict") + elif not isinstance(tables, dict): + tables = {name: tables} + for name, df in tables.items(): + if not (isinstance(df, pd.DataFrame) or isinstance(df, pd.Series)): + raise ValueError( + "table type not recognized, should be pandas DataFrame or Series." + ) + if name in self.tables: + if not self._write: + raise IOError(f"Cannot overwrite table {name} in read-only mode") + elif self._read: + self.logger.warning(f"Overwriting table: {name}") - def set_tables(self, table_dict) -> None: - """Add a table to model.""" - for name, df in table_dict.items(): - self.set_table(name, df) + self.tables[name] = df def get_tables_merged(self) -> pd.DataFrame: - """Return all tables of a model merged into one dataframe. - - This is mostly used for convinience and testing. - """ + """Return all tables of a model merged into one dataframe.""" + # This is mostly used for convenience and testing. return pd.concat( [df.assign(table_origin=name) for name, df in self.tables.items()], axis=0 ) diff --git a/hydromt/models/model_grid.py b/hydromt/models/model_grid.py index c081d1320..efb4376b8 100644 --- a/hydromt/models/model_grid.py +++ b/hydromt/models/model_grid.py @@ -683,6 +683,7 @@ def read( "config", "grid", "geoms", + "tables", "forcing", "states", "results", @@ -695,13 +696,21 @@ def read( components : List, optional List of model components to read, each should have an associated read_ method. By default ['config', 'maps', 'grid', - 'geoms', 'forcing', 'states', 'results'] + 'geoms', 'tables', 'forcing', 'states', 'results'] """ super().read(components=components) def write( self, - components: List = ["config", "maps", "grid", "geoms", "forcing", "states"], + components: List = [ + "config", + "maps", + "grid", + "geoms", + "tables", + "forcing", + "states", + ], ) -> None: """Write the complete model schematization and configuration to model files. @@ -709,8 +718,8 @@ def write( ---------- components : List, optional List of model components to write, each should have an - associated write_ method. - By default ['config', 'maps', 'grid', 'geoms', 'forcing', 'states'] + associated write_ method. By default + ['config', 'maps', 'grid', 'geoms', 'tables', 'forcing', 'states'] """ super().write(components=components) diff --git a/hydromt/models/model_lumped.py b/hydromt/models/model_lumped.py index dd72a9f59..a18302536 100644 --- a/hydromt/models/model_lumped.py +++ b/hydromt/models/model_lumped.py @@ -180,6 +180,7 @@ def read( "config", "response_units", "geoms", + "tables", "forcing", "states", "results", @@ -192,7 +193,7 @@ def read( components : List, optional List of model components to read, each should have an associated read_ method. - By default ['config', 'maps', 'response_units', 'geoms', + By default ['config', 'maps', 'response_units', 'geoms', 'tables', 'forcing', 'states', 'results'] """ super().read(components=components) @@ -203,6 +204,7 @@ def write( "config", "response_units", "geoms", + "tables", "forcing", "states", ], @@ -214,7 +216,7 @@ def write( components : List, optional List of model components to write, each should have an associated write_ method. By default ['config', - 'maps', 'response_units', 'geoms', 'forcing', 'states'] + 'maps', 'response_units', 'geoms', 'tables', 'forcing', 'states'] """ super().write(components=components) diff --git a/hydromt/models/model_mesh.py b/hydromt/models/model_mesh.py index dbd5c1ab7..02165d964 100644 --- a/hydromt/models/model_mesh.py +++ b/hydromt/models/model_mesh.py @@ -647,6 +647,7 @@ def read( "config", "mesh", "geoms", + "tables", "forcing", "states", "results", @@ -665,7 +666,7 @@ def read( def write( self, - components: List = ["config", "mesh", "geoms", "forcing", "states"], + components: List = ["config", "mesh", "geoms", "tables", "forcing", "states"], ) -> None: """Write the complete model schematization and configuration to model files. @@ -674,7 +675,7 @@ def write( components : List, optional List of model components to write, each should have an associated write_ method. By default ['config', 'maps', - 'mesh', 'geoms', 'forcing', 'states'] + 'mesh', 'geoms', 'tables', 'forcing', 'states'] """ super().write(components=components) diff --git a/hydromt/models/model_network.py b/hydromt/models/model_network.py index 3e6c32bca..1fb385b5e 100644 --- a/hydromt/models/model_network.py +++ b/hydromt/models/model_network.py @@ -46,6 +46,7 @@ def read( "config", "network", "geoms", + "tables", "forcing", "states", "results", @@ -58,7 +59,7 @@ def read( components : List, optional List of model components to read, each should have an associated read_ method. By default ['config', 'maps', - 'network', 'geoms', 'forcing', 'states', 'results'] + 'network', 'geoms', 'tables', 'forcing', 'states', 'results'] """ super().read(components=components) @@ -68,6 +69,7 @@ def write( "config", "network", "geoms", + "tables", "forcing", "states", ], @@ -79,7 +81,7 @@ def write( components : List, optional List of model components to write, each should have an associated write_ method. By default ['config', 'maps', - 'network', 'geoms', 'forcing', 'states'] + 'network', 'geoms', 'tables', 'forcing', 'states'] """ super().write(components=components) diff --git a/tests/test_model.py b/tests/test_model.py index cde3bf82a..7d72c713a 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -182,12 +182,12 @@ def test_model_tables(model, df, tmpdir): model.tables[1] for i, d in dfs.items(): - model.set_table(i, d) + model.set_tables(d, name=i) assert df.equals(model.tables[i]) # now do the same but interating over the stables instead for i, d in model.tables.items(): - model.set_table(i, d) + model.set_tables(d, name=i) assert df.equals(model.tables[i]) assert list(model.tables.keys()) == list(map(str, range(5))) @@ -204,7 +204,7 @@ def test_model_tables(model, df, tmpdir): ), f"model: {model_merged}\nclean_model: {clean_model_merged}" -def test_model_append(demda, tmpdir): +def test_model_append(demda, df, tmpdir): # write a model demda.name = "dem" mod = GridModel(mode="w", root=str(tmpdir)) @@ -214,6 +214,7 @@ def test_model_append(demda, tmpdir): mod.set_forcing(demda, name="dem") mod.set_states(demda, name="dem") mod.set_geoms(demda.raster.box, name="dem") + mod.set_tables(df, name="df") mod.write() # append to model and check if previous data is still there mod1 = GridModel(mode="r+", root=str(tmpdir)) @@ -229,6 +230,8 @@ def test_model_append(demda, tmpdir): assert "dem" in mod1.states mod1.set_geoms(demda.raster.box, name="dem1") assert "dem" in mod1.geoms + mod1.set_tables(df, name="df1") + assert "df" in mod1.tables @pytest.mark.filterwarnings("ignore:The setup_basemaps") From 9ba4cd6d52950f00d71216f69b6ca05a434944c9 Mon Sep 17 00:00:00 2001 From: Dirk Eilander Date: Thu, 7 Sep 2023 15:48:19 +0200 Subject: [PATCH 5/6] add tables to api --- hydromt/models/model_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydromt/models/model_api.py b/hydromt/models/model_api.py index 007a082d2..71c47bf9b 100644 --- a/hydromt/models/model_api.py +++ b/hydromt/models/model_api.py @@ -61,6 +61,7 @@ class Model(object, metaclass=ABCMeta): "crs": CRS, "config": Dict[str, Any], "geoms": Dict[str, gpd.GeoDataFrame], + "tables": Dict[str, pd.DataFrame], "maps": XArrayDict, "forcing": XArrayDict, "region": gpd.GeoDataFrame, @@ -110,7 +111,6 @@ def __init__( # placeholders # metadata maps that can be at different resolutions - # TODO do we want read/write maps? self._config = None # nested dictionary self._maps: Optional[XArrayDict] = None self._tables: Dict[str, pd.DataFrame] = None From cee1f6806e001060a71eda6645532d8a36e9f1c8 Mon Sep 17 00:00:00 2001 From: Dirk Eilander Date: Thu, 7 Sep 2023 16:08:30 +0200 Subject: [PATCH 6/6] fix test --- tests/test_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_model.py b/tests/test_model.py index 7d72c713a..8558b9779 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -175,7 +175,7 @@ def test_model(model, tmpdir): def test_model_tables(model, df, tmpdir): # make a couple copies of the dfs for testing dfs = {str(i): df.copy() for i in range(5)} - model.set_root(tmpdir) + model.set_root(tmpdir, mode="r+") # append mode clean_model = deepcopy(model) with pytest.raises(KeyError):