From ebbd000b883a667d4a58ef3e43fbfb967006782e Mon Sep 17 00:00:00 2001 From: liellnima Date: Tue, 4 Jun 2024 11:52:29 +0200 Subject: [PATCH 1/6] add config folders and empty config files --- config/README.md | 18 ++++++++++++++++++ config/cdo/cdo.yaml | 1 + config/cmip6_processer/cmip6_processor.yaml | 0 config/downloader/downloader_configs.yaml | 0 config/esgf_server/esgf_server.yaml | 0 .../input4mips_processor.yaml | 0 config/main_config.yaml | 4 ++++ config/resolutions/resolutions.yaml | 5 +++++ config/units/units.yaml | 3 +++ 9 files changed, 31 insertions(+) create mode 100644 config/cdo/cdo.yaml create mode 100644 config/cmip6_processer/cmip6_processor.yaml create mode 100644 config/downloader/downloader_configs.yaml create mode 100644 config/esgf_server/esgf_server.yaml create mode 100644 config/input4mips_processer/input4mips_processor.yaml create mode 100644 config/main_config.yaml create mode 100644 config/resolutions/resolutions.yaml create mode 100644 config/units/units.yaml diff --git a/config/README.md b/config/README.md index caf7a29..b435cc9 100644 --- a/config/README.md +++ b/config/README.md @@ -6,3 +6,21 @@ This folder is for configurations, like: - Training parameters - Script parameters - etc. + +The following configs need to be adapted by the user: +- main_config.yaml +- downloader +- cmip6 processor +- input4mips processor + +If you are adding a new variables, please edit: +- units + +If you are using a new output resolution (i.e. you are not using NorESM as "role model" for the output resolution), please edit: +- resolutions +- output_rolemodel + +## Output Role Model +This is the only config directory that has not a yaml file, but a netcdf file. Please store here the role model of your desired output data. The default is NorESM, i.e. all files will be aligned to have the same resolution format like NorESM (in terms of where longitude / latitude points start, step size, etc.). + + diff --git a/config/cdo/cdo.yaml b/config/cdo/cdo.yaml new file mode 100644 index 0000000..cee6451 --- /dev/null +++ b/config/cdo/cdo.yaml @@ -0,0 +1 @@ +# add cdo constants \ No newline at end of file diff --git a/config/cmip6_processer/cmip6_processor.yaml b/config/cmip6_processer/cmip6_processor.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/downloader/downloader_configs.yaml b/config/downloader/downloader_configs.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/esgf_server/esgf_server.yaml b/config/esgf_server/esgf_server.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/input4mips_processer/input4mips_processor.yaml b/config/input4mips_processer/input4mips_processor.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/main_config.yaml b/config/main_config.yaml new file mode 100644 index 0000000..70f9f8a --- /dev/null +++ b/config/main_config.yaml @@ -0,0 +1,4 @@ + +# TODO add data paths here? + +# TODO link all the other configs \ No newline at end of file diff --git a/config/resolutions/resolutions.yaml b/config/resolutions/resolutions.yaml new file mode 100644 index 0000000..4b3625f --- /dev/null +++ b/config/resolutions/resolutions.yaml @@ -0,0 +1,5 @@ +# TODO add remapping + +# TODO add calendar + +# TODO add long / lats (min, max, step) \ No newline at end of file diff --git a/config/units/units.yaml b/config/units/units.yaml new file mode 100644 index 0000000..3a9aa02 --- /dev/null +++ b/config/units/units.yaml @@ -0,0 +1,3 @@ +# add units of all variables + +# TODO every time you use a new variable it needs to be added here \ No newline at end of file From 032fa44ffe05ff88c5ebe4607acfb9f6876e82d3 Mon Sep 17 00:00:00 2001 From: liellnima Date: Tue, 4 Jun 2024 11:53:30 +0200 Subject: [PATCH 2/6] add comment for including role model netcdf files. fails large file check, which is why it's not used yet --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f8af546..35633a1 100644 --- a/.gitignore +++ b/.gitignore @@ -166,8 +166,10 @@ cython_debug/ data/* !data/README.md **/*.nc +#!config/output_rolemodel/*.nc **/*.tif + # Makefile and configurations Makefile.private scripts/config.sh From 9de181fac5ecf928a3801341a25a14a68b7eca87 Mon Sep 17 00:00:00 2001 From: liellnima Date: Tue, 4 Jun 2024 11:54:01 +0200 Subject: [PATCH 3/6] add script dirs and empty examples --- scripts/cdo/get_co2_data.sh | 0 scripts/cdo/prepare_CH4.sh | 0 scripts/downloader/downloader.sh | 0 scripts/processors/cmip6_models/cc_cesm2-waccm.sh | 0 scripts/processors/cmip6_models/cc_ec-earth3-veg.sh | 0 scripts/processors/cmip6_models/cc_miroc6.sh | 0 scripts/processors/examples/cc_cmip6_example.sh | 0 scripts/processors/examples/cc_input4mips_example.sh | 0 scripts/processors/examples/mila_cmip6_example.sh | 0 scripts/processors/examples/mila_input4mips_example.sh | 0 scripts/processors/input4mips/cc_input4mips.sh | 0 11 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/cdo/get_co2_data.sh create mode 100644 scripts/cdo/prepare_CH4.sh create mode 100644 scripts/downloader/downloader.sh create mode 100644 scripts/processors/cmip6_models/cc_cesm2-waccm.sh create mode 100644 scripts/processors/cmip6_models/cc_ec-earth3-veg.sh create mode 100644 scripts/processors/cmip6_models/cc_miroc6.sh create mode 100644 scripts/processors/examples/cc_cmip6_example.sh create mode 100644 scripts/processors/examples/cc_input4mips_example.sh create mode 100644 scripts/processors/examples/mila_cmip6_example.sh create mode 100644 scripts/processors/examples/mila_input4mips_example.sh create mode 100644 scripts/processors/input4mips/cc_input4mips.sh diff --git a/scripts/cdo/get_co2_data.sh b/scripts/cdo/get_co2_data.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/cdo/prepare_CH4.sh b/scripts/cdo/prepare_CH4.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/downloader/downloader.sh b/scripts/downloader/downloader.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/cmip6_models/cc_cesm2-waccm.sh b/scripts/processors/cmip6_models/cc_cesm2-waccm.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/cmip6_models/cc_ec-earth3-veg.sh b/scripts/processors/cmip6_models/cc_ec-earth3-veg.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/cmip6_models/cc_miroc6.sh b/scripts/processors/cmip6_models/cc_miroc6.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/examples/cc_cmip6_example.sh b/scripts/processors/examples/cc_cmip6_example.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/examples/cc_input4mips_example.sh b/scripts/processors/examples/cc_input4mips_example.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/examples/mila_cmip6_example.sh b/scripts/processors/examples/mila_cmip6_example.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/examples/mila_input4mips_example.sh b/scripts/processors/examples/mila_input4mips_example.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/processors/input4mips/cc_input4mips.sh b/scripts/processors/input4mips/cc_input4mips.sh new file mode 100644 index 0000000..e69de29 From 3314a838071183adef83b8354e8b6cca5fdb7ee4 Mon Sep 17 00:00:00 2001 From: liellnima Date: Tue, 4 Jun 2024 11:54:21 +0200 Subject: [PATCH 4/6] add empty test files --- tests/test_cmip6_processor.py | 0 tests/test_downloader.py | 0 tests/test_input4mips_processor.py | 0 tests/test_res_processor.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_cmip6_processor.py create mode 100644 tests/test_downloader.py create mode 100644 tests/test_input4mips_processor.py create mode 100644 tests/test_res_processor.py diff --git a/tests/test_cmip6_processor.py b/tests/test_cmip6_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_downloader.py b/tests/test_downloader.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_input4mips_processor.py b/tests/test_input4mips_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_res_processor.py b/tests/test_res_processor.py new file mode 100644 index 0000000..e69de29 From 957ddb8e89c11db70fd96e3773b7969bb21df03a Mon Sep 17 00:00:00 2001 From: liellnima Date: Tue, 4 Jun 2024 11:55:09 +0200 Subject: [PATCH 5/6] add structure for data builders --- climateset/downloader/downloader.py | 0 climateset/raw/cmip6_processor.py | 0 climateset/raw/input4mips_processor.py | 0 climateset/raw/raw_processor.py | 0 climateset/resolution/resolution_processor.py | 0 climateset/resolution/spatial_resolution_processor.py | 0 climateset/resolution/temporal_resolution_processor.py | 0 climateset/run.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 climateset/downloader/downloader.py create mode 100644 climateset/raw/cmip6_processor.py create mode 100644 climateset/raw/input4mips_processor.py create mode 100644 climateset/raw/raw_processor.py create mode 100644 climateset/resolution/resolution_processor.py create mode 100644 climateset/resolution/spatial_resolution_processor.py create mode 100644 climateset/resolution/temporal_resolution_processor.py create mode 100644 climateset/run.py diff --git a/climateset/downloader/downloader.py b/climateset/downloader/downloader.py new file mode 100644 index 0000000..e69de29 diff --git a/climateset/raw/cmip6_processor.py b/climateset/raw/cmip6_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/climateset/raw/input4mips_processor.py b/climateset/raw/input4mips_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/climateset/raw/raw_processor.py b/climateset/raw/raw_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/climateset/resolution/resolution_processor.py b/climateset/resolution/resolution_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/climateset/resolution/spatial_resolution_processor.py b/climateset/resolution/spatial_resolution_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/climateset/resolution/temporal_resolution_processor.py b/climateset/resolution/temporal_resolution_processor.py new file mode 100644 index 0000000..e69de29 diff --git a/climateset/run.py b/climateset/run.py new file mode 100644 index 0000000..e69de29 From c99f9d2df7c54e3e71bc1e53cf2e5853a1957b2e Mon Sep 17 00:00:00 2001 From: liellnima Date: Tue, 16 Jul 2024 17:45:55 +0200 Subject: [PATCH 6/6] delete unnecessary file --- climateset/run.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 climateset/run.py diff --git a/climateset/run.py b/climateset/run.py deleted file mode 100644 index e69de29..0000000