198 narclim (#239)

* Added Narclim metadata, translator & tests for translator
ACCESS-NRI · Nov 1, 2024 · 2323b3a · 2323b3a
1 parent 68382fe
commit 2323b3a
Show file tree

Hide file tree

Showing 6 changed files with 361 additions and 0 deletions.
diff --git a/config/metadata_sources/narclim2-zz63/metadata.yaml b/config/metadata_sources/narclim2-zz63/metadata.yaml
@@ -0,0 +1,210 @@
+name: narclim2_zz63
+experiment_uuid: e238a14d-d3df-43f3-b7ff-fd247b03816e
+description: <REQUIRED Short description of the experiment (string, < 150 char)>
+long_description: >-
+  NARCliM2.0 climate projections dynamically downscaled from ACCESS-ESM1-5 r6i1p1f1 
+  historical model using the WRF regional model version NARClIM2-0-WRF412R3. Domain 
+  covers a region encompassing CORDEX Australasia (AUS-18) at approximately 18km 
+  resolution in rotated grid format. The dataset includes 1-hour, 3-hour, daily 
+  and monthly data for variables such as temperature, humidity, wind, radiation 
+  and precipitation parameters. Data are provided at 1-hour steps for surface-level 
+  variables, other data at atmospheric pressure levels or subsurface processes is 
+  provided as 3-hour, daily and monthly data. Dataset includes all of CORDEX Core 
+  variables, most of the CORDEX Tier 1 and just over half of CORDEX Tier 2 variables.
+model:
+- MPI-ESM1-2-HR
+- UKESM1-0-LL
+- ACCESS-ESM1-5
+- NorESM2-MM
+- EC-Earth3-Veg
+realm:
+- atmos
+frequency:
+- 1hr
+- mon
+- day
+- 3hr
+- fx
+variable:
+- evspsbl
+- rsus
+- va100
+- va250m
+- snc
+- hus600
+- CIN
+- zg850
+- hus70
+- va200
+- zg150
+- tasmax
+- hus850
+- ua200
+- ta70
+- va250
+- tas
+- ta925
+- wa250
+- wa300
+- lai
+- wa500
+- va150
+- va1000
+- ua1000
+- vegFrac
+- ua200m
+- mrfso
+- hus100
+- hus250
+- rlus
+- siconca
+- zg600
+- wa700
+- ta700
+- sfcWindmax
+- va400
+- hus1000
+- vas
+- snd
+- snm
+- ts
+- ua850
+- tasmin
+- wa600
+- wa400
+- ua750
+- hfss
+- va700
+- hus200
+- va150m
+- ua500
+- zg750
+- rsds
+- va600
+- va300
+- mrfsos
+- wa1000
+- ua100
+- ua700
+- rsdt
+- rsdscs
+- prw
+- ua250m
+- rsdsdir
+- ta300
+- CAPEmax
+- va925
+- zg500
+- ta50m
+- tsl
+- va100m
+- CINmax
+- tauu
+- ta750
+- ua100m
+- rsutcs
+- va70
+- hus150
+- zg70
+- wa100
+- ua50m
+- hus400
+- ua250
+- wa925
+- rlut
+- zg300
+- wa150
+- hus300m
+- va300m
+- ua150
+- zg400
+- rldscs
+- zg700
+- ua600
+- clt
+- va500
+- hus925
+- prhmax
+- zg100
+- ta200
+- wa70
+- ta600
+- wa850
+- clwvi
+- ua300m
+- zg1000
+- prsn
+- ta400
+- rlutcs
+- hus750
+- hus500
+- rlds
+- va200m
+- wa200
+- mrsos
+- ua300
+- ta1000
+- zg925
+- rsut
+- huss
+- mrros
+- ta850
+- mrso
+- clivi
+- ua400
+- ta100
+- hus700
+- zg200
+- psl
+- rluscs
+- ta500
+- zg250
+- CAPE
+- wa750
+- snw
+- rsuscs
+- va850
+- va750
+- hus300
+- hus50m
+- ua150m
+- ta250
+- hfls
+- zmla
+- sfcWind
+- tauv
+- ps
+- ta150
+- ua70
+- ua925
+- ta300m
+- hurs
+- uas
+- va50m
+- pr
+- mrro
+- prc
+- areacella
+- orog
+- sftlf
+nominal_resolution:
+- 18km
+version: 2.0
+contact: NCI
+email: [email protected]
+created: 2024-07-09
+reference: >-
+- https://doi.org/10.1029/2021EF002625
+- https://doi.org/10.5194/gmd-2024-41
+- https://doi.org/10.5194/gmd-2024-87
+license: Creative Commons Attribution 4.0 International
+url: https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f7334_2646_2487_9846
+parent_experiment: CMIP6
+related_experiments:
+notes: 
+keywords:
+- CMIP6
+- CORDEX
+- NARCliM, 
+- GCM 
+- RCM
diff --git a/config/narclim2.yaml b/config/narclim2.yaml
@@ -0,0 +1,9 @@
+builder: null
+
+translator: NarclimTranslator
+
+sources:
+
+  - metadata_yaml: /g/data/xp65/admin/intake/metadata/narclim2-zz63/metadata.yaml
+    path:
+      - g/data/zz63/catalog/v2/esm/catalog.json
diff --git a/src/access_nri_intake/catalog/translators.py b/src/access_nri_intake/catalog/translators.py
@@ -530,6 +530,48 @@ def _realm_translator(self):
         return self.source.df.apply(lambda x: ("none",), 1)
 
 
+class NarclimTranslator(DefaultTranslator):
+    def __init__(self, source, columns):
+        """
+        Initialise a BarpaTranslator
+
+        Parameters
+        ----------
+        source: :py:class:`~intake.DataSource`
+            The NCI BARPA intake-esm datastore
+        columns: list of str
+            The columns to translate to (these are the core columns in the intake-dataframe-catalog)
+        """
+
+        super().__init__(source, columns)
+        self.set_dispatch(
+            input_name="source_id",
+            core_colname="model",
+            func=super()._model_translator,
+        )
+        self.set_dispatch(
+            input_name="realm",
+            core_colname="realm",
+            func=self._realm_translator,
+        )
+        self.set_dispatch(
+            input_name="frequency",
+            core_colname="frequency",
+            func=super()._frequency_translator,
+        )
+        self.set_dispatch(
+            input_name="variable_id",
+            core_colname="variable",
+            func=super()._variable_translator,
+        )
+
+    def _realm_translator(self):
+        """
+        Return realm, fixing a few issues
+        """
+        return self.source.df.apply(lambda x: ("atmos",), 1)
+
+
 @dataclass
 class _DispatchKeys:
     """

diff --git a/tests/data/esm_datastore/narclim2-zz63.csv b/tests/data/esm_datastore/narclim2-zz63.csv
@@ -0,0 +1,6 @@
+,path,file_type,project_id,institution_id,source_id,experiment_id,member_id,resolution,resolution_version,frequency,variable_id,version,time_range
+0,/g/data/zz63/NARCliM2-0/output/CMIP6/DD/NARCliM2-0-SEAus-04/NSW-Government/ACCESS-ESM1-5/ssp126/r6i1p1f1/NARCliM2-0-WRF412R3/v1-r1/mon/va100m/v20240312/va100m_NARCliM2-0-SEAus-04_ACCESS-ESM1-5_ssp126_r6i1p1f1_NSW-Government_NARCliM2-0-WRF412R3_v1-r1_mon_203201-203212.nc,f,NARCliM2-0-SEAus-04,NSW-Government,ACCESS-ESM1-5,ssp126,r6i1p1f1,NARCliM2-0-WRF412R3,v1-r1,mon,va100m,v20240312,203201-203212
+1,/g/data/zz63/NARCliM2-0/output/CMIP6/DD/AUS-18/NSW-Government/ACCESS-ESM1-5/historical/r6i1p1f1/NARCliM2-0-WRF412R3/v1-r1/3hr/ua100m/v20240312/ua100m_AUS-18_ACCESS-ESM1-5_historical_r6i1p1f1_NSW-Government_NARCliM2-0-WRF412R3_v1-r1_3hr_1965010100-1965123121.nc,f,AUS-18,NSW-Government,ACCESS-ESM1-5,historical,r6i1p1f1,NARCliM2-0-WRF412R3,v1-r1,3hr,ua100m,v20240312,1965010100-1965123121
+2,/g/data/zz63/NARCliM2-0/output/CMIP6/DD/AUS-18/NSW-Government/UKESM1-0-LL/ssp126/r1i1p1f2/NARCliM2-0-WRF412R5/v1-r1/3hr/zg700/v20240827/zg700_AUS-18_UKESM1-0-LL_ssp126_r1i1p1f2_NSW-Government_NARCliM2-0-WRF412R5_v1-r1_3hr_2093010100-2093123021.nc,f,AUS-18,NSW-Government,UKESM1-0-LL,ssp126,r1i1p1f2,NARCliM2-0-WRF412R5,v1-r1,3hr,zg700,v20240827,2093010100-2093123021
+3,/g/data/zz63/NARCliM2-0/output/CMIP6/DD/AUS-18/NSW-Government/UKESM1-0-LL/historical/r1i1p1f2/NARCliM2-0-WRF412R3/v1-r1/fx/sftlf/v20240312/sftlf_AUS-18_UKESM1-0-LL_historical_r1i1p1f2_NSW-Government_NARCliM2-0-WRF412R3_v1-r1_fx.nc,f,AUS-18,NSW-Government,UKESM1-0-LL,historical,r1i1p1f2,NARCliM2-0-WRF412R3,v1-r1,fx,sftlf,v20240312,na
+4,/g/data/zz63/NARCliM2-0/output/CMIP6/DD/AUS-18/NSW-Government/UKESM1-0-LL/ssp370/r1i1p1f2/NARCliM2-0-WRF412R5/v1-r1/mon/va100m/v20240312/va100m_AUS-18_UKESM1-0-LL_ssp370_r1i1p1f2_NSW-Government_NARCliM2-0-WRF412R5_v1-r1_mon_206201-206212.nc,f,AUS-18,NSW-Government,UKESM1-0-LL,ssp370,r1i1p1f2,NARCliM2-0-WRF412R5,v1-r1,mon,va100m,v20240312,206201-206212
diff --git a/tests/data/esm_datastore/narclim2-zz63.json b/tests/data/esm_datastore/narclim2-zz63.json
@@ -0,0 +1,74 @@
+{
+    "id": "narclim2-zz63",
+    "title": "narclim2-zz63",
+    "description": "Datasets on Gadi, both publised and replicated. All file versions present are in the listing\nMaintained By: NCI\nContact: [email protected]",
+    "assets": {
+        "column_name": "path",
+        "format": "netcdf"
+    },
+    "aggregation_control": {
+        "variable_column_name": "variable_id",
+        "groupby_attrs": [
+            "file_type",
+            "project_id",
+            "institution_id",
+            "source_id",
+            "experiment_id",
+            "member_id",
+            "resolution",
+            "resolution_version",
+            "frequency",
+            "variable_id",
+            "version"
+        ],
+        "aggregations": [
+            {
+                "type": "join_existing",
+                "attribute_name": "time_range",
+                "options": {
+                    "dim": "time"
+                }
+            }
+        ]
+    },
+    "esmcat_version": "0.1.0",
+    "catalog_file": "narclim2-zz63.csv",
+    "attributes": [
+        {
+            "column_name": "file_type"
+        },
+        {
+            "column_name": "project_id"
+        },
+        {
+            "column_name": "institution_id"
+        },
+        {
+            "column_name": "source_id"
+        },
+        {
+            "column_name": "experiment_id"
+        },
+        {
+            "column_name": "member_id"
+        },
+        {
+            "column_name": "resolution"
+        },
+        {
+            "column_name": "resolution_version"
+        },
+        {
+            "column_name": "frequency"
+        },
+        {
+            "column_name": "variable_id"
+        },
+        {
+            "column_name": "version"
+        },
+        {
+            "column_name": "time_range"
+        }
+    ]
+}
diff --git a/tests/test_translators.py b/tests/test_translators.py
@@ -15,6 +15,7 @@
     CordexTranslator,
     DefaultTranslator,
     Era5Translator,
+    NarclimTranslator,
     TranslatorError,
     _cmip_realm_translator,
     _to_tuple,
@@ -341,3 +342,22 @@ def test_CcamTranslator(test_data, groupby, n_entries):
     esmds.description = "description"
     df = CcamTranslator(esmds, CORE_COLUMNS).translate(groupby)
     assert len(df) == n_entries
+
+
+@pytest.mark.parametrize(
+    "groupby, n_entries",
+    [
+        (None, 5),
+        (["variable"], 4),
+        (["frequency"], 3),
+        (["model"], 2),
+        (["realm"], 1),
+    ],
+)
+def test_NarclimTranslator(test_data, groupby, n_entries):
+    """Test ERA5 datastore translator"""
+    esmds = intake.open_esm_datastore(test_data / "esm_datastore/narclim2-zz63.json")
+    esmds.name = "name"
+    esmds.description = "description"
+    df = NarclimTranslator(esmds, CORE_COLUMNS).translate(groupby)
+    assert len(df) == n_entries