use recent metacatalog to load combined model

Dockerfile

@@ -1,11 +1,16 @@
 # Pull any base image that includes python3
 FROM python:3.12.2
 
+# Install GDAL which will be used by geopandas
+RUN pip install --upgrade pip
+RUN apt-get update && apt-get install -y gdal-bin libgdal-dev
+RUN pip install GDAL==$(gdal-config --version | awk -F'[.]' '{print $1"."$2}')
+
 # install the toolbox runner tools
-RUN pip install "json2args>=0.6.2" \
+RUN pip install "json2args>=0.7.0" \
     metacatalog==0.9.2 \    
     ipython==8.26.0 \ 
-    pandas==2.2.2 \
+    pandas==2.1.4 \
     geopandas==1.0.1 \
     xarray[complete]==2024.7.0 \ 
     rioxarray==0.17.0 \

src/utils.py

@@ -0,0 +1,37 @@
+from typing import TypedDict, List
+from pathlib import Path
+import warnings
+
+from metacatalog.models.entry import Entry
+from metacatalog.models.combined import Metadata, from_file
+
+
+class FileMapping(TypedDict):
+    entry: Entry | Metadata
+    data_path: str
+
+
+def get_file_mapping(datasets_path: str, metadata_files: str = '*.metadata.json') -> List[FileMapping]:
+    file_mapping: List[FileMapping] = []
+
+    # iterate over all metadata files
+    for meta_file in Path(datasets_path).rglob(metadata_files):
+        # load the metadata
+        meta = from_file(meta_file)
+
+        # try to find a dataset folder
+        folder_name = meta_file.name.replace(metadata_files.replace('*', ''), '')
+        data_path = (meta_file.parent / folder_name).resolve()
+
+        # check there are files with that stem
+        if len(list(data_path.glob('*'))) > 0:
+            file_mapping.append({
+                'entry': meta,
+                'data_path': str(data_path)
+            })
+        else:
+            warnings.warn(f"Found metadata file {meta_file}, but no associated dataset folder: {data_path / '*'}")
+
+
+    return file_mapping
+