ENH: data versioning in the data catalog

docs/changelog.rst

@@ -11,7 +11,7 @@ Unreleased
 
 Added
 -----
--
+- Support for loading the same data source but from different places (e.g. local & aws)
 
 Changed
 -------

docs/user_guide/data_prepare_cat.rst

@@ -114,6 +114,8 @@ A full list of **optional data source arguments** is given below
 - **placeholder** (optional): this argument can be used to generate multiple sources with a single entry in the data catalog file. If different files follow a logical
   nomenclature, multiple data sources can be defined by iterating through all possible combinations of the placeholders. The placeholder names should be given in the
   source name and the path and its values listed under the placeholder argument.
+- **versions** (optional): If you want to use the same data source but load it from different places (e.g. local & aws) you can add this key
+  Keys here are essentially overrides that will get applied to the containing catalog when they get parsed and expanded.
 
 .. note::
 

hydromt/__init__.py

@@ -26,3 +26,4 @@
 
 # high-level methods
 from .models import *
+from .utils import *

hydromt/cli/api.py

@@ -100,13 +100,12 @@ def get_datasets(data_libs: Union[List, str]) -> Dict:
         for accepted yaml format.
     """
     data_catalog = DataCatalog(data_libs)
-    datasets = data_catalog.sources
     dataset_sources = {
         "RasterDatasetSource": [],
         "GeoDatasetSource": [],
         "GeoDataframeSource": [],
     }
-    for k, v in datasets.items():
+    for k, v in data_catalog.iter_sources():
         if v.data_type == "RasterDataset":
             dataset_sources["RasterDatasetSource"].append(k)
         elif v.data_type == "GeoDataFrame":
@@ -167,7 +166,7 @@ def get_region(
         # retrieve global hydrography data (lazy!)
         ds_org = data_catalog.get_rasterdataset(hydrography_fn)
         if "bounds" not in region:
-            region.update(basin_index=data_catalog[basin_index_fn])
+            region.update(basin_index=data_catalog.get_source(basin_index_fn))
         # get basin geometry
         geom, xy = workflows.get_basin_geometry(
             ds=ds_org,

hydromt/data_adapter/data_adapter.py

@@ -125,6 +125,7 @@ def __init__(
         driver_kwargs={},
         name="",  # optional for now
         catalog_name="",  # optional for now
+        version_name=None,
     ):
         """General Interface to data source for HydroMT.
 
@@ -170,6 +171,7 @@ def __init__(
         """
         self.name = name
         self.catalog_name = catalog_name
+        self.version_name = version_name
         # general arguments
         self.path = path
         # driver and driver keyword-arguments

hydromt/data_adapter/dataframe.py

@@ -38,6 +38,7 @@ def __init__(
         driver_kwargs: dict = {},
         name: str = "",  # optional for now
         catalog_name: str = "",  # optional for now
+        version_name=None,
         **kwargs,
     ):
         """Initiate data adapter for 2D tabular data.
@@ -106,6 +107,7 @@ def __init__(
             driver_kwargs=driver_kwargs,
             name=name,
             catalog_name=catalog_name,
+            version_name=version_name,
         )
 
     def to_file(

hydromt/data_adapter/geodataframe.py

@@ -46,6 +46,7 @@ def __init__(
         driver_kwargs: dict = {},
         name: str = "",  # optional for now
         catalog_name: str = "",  # optional for now
+        version_name=None,
         **kwargs,
     ):
         """Initiate data adapter for geospatial vector data.
@@ -116,6 +117,7 @@ def __init__(
             driver_kwargs=driver_kwargs,
             name=name,
             catalog_name=catalog_name,
+            version_name=version_name,
         )
         self.crs = crs
 

hydromt/data_adapter/geodataset.py

@@ -47,6 +47,7 @@ def __init__(
         driver_kwargs: dict = {},
         name: str = "",  # optional for now
         catalog_name: str = "",  # optional for now
+        version_name=None,
         **kwargs,
     ):
         """Initiate data adapter for geospatial timeseries data.
@@ -123,6 +124,7 @@ def __init__(
             driver_kwargs=driver_kwargs,
             name=name,
             catalog_name=catalog_name,
+            version_name=version_name,
         )
         self.crs = crs
 

hydromt/data_adapter/rasterdataset.py

@@ -50,6 +50,7 @@ def __init__(
         zoom_levels: dict = {},
         name: str = "",  # optional for now
         catalog_name: str = "",  # optional for now
+        version_name=None,
         **kwargs,
     ):
         """Initiate data adapter for geospatial raster data.
@@ -127,6 +128,7 @@ def __init__(
             driver_kwargs=driver_kwargs,
             name=name,
             catalog_name=catalog_name,
+            version_name=version_name,
         )
         self.crs = crs
         self.zoom_levels = zoom_levels