From edbd0c9c4bdb9cded2439f48534da7b86fe5140a Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 18 Dec 2024 10:54:31 -0600 Subject: [PATCH 01/18] resources, no gfs, gdas dev --- ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml | 2 +- parm/config/gfs/config.resources | 38 +++++++++------------- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 2 +- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index 4147249a4c..c47b53339e 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 384 resdetocean: 0.25 nens: 0 - interval: 6 + interval: 0 start: warm comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index e642082290..f17eb7cc50 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -502,16 +502,14 @@ case ${step} in "marineanlinit") walltime="00:10:00" ntasks=1 - threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} memory="24GB" ;; "prepoceanobs") walltime="00:10:00" ntasks=1 - threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} memory="48GB" ;; @@ -519,7 +517,9 @@ case ${step} in npes=16 ntasks=16 case ${OCNRES} in - "025") ntasks=480;; + "025") + ntasks=480;; + memory="256GB" "050") ntasks=16;; "100") ntasks=16;; "500") ntasks=16;; @@ -529,9 +529,8 @@ case ${step} in esac walltime="00:30:00" - threads_per_task=1 export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; "marineanlvar") @@ -539,7 +538,7 @@ case ${step} in case ${OCNRES} in "025") ntasks=480 - memory="96GB" + memory="256GB" ;; "050") ntasks=16 @@ -558,10 +557,9 @@ case ${step} in exit 4 esac - walltime="00:15:00" - threads_per_task=1 + walltime="00:30:00" export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; "ocnanalecen") @@ -569,7 +567,7 @@ case ${step} in case ${OCNRES} in "025") ntasks=40 - memory="96GB" + memory="256GB" ;; "050") ntasks=16 @@ -589,9 +587,8 @@ case ${step} in esac walltime="00:10:00" - threads_per_task=1 export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; "marineanlletkf") @@ -599,7 +596,7 @@ case ${step} in case ${OCNRES} in "025") ntasks=480 - memory="96GB" + memory="256GB" ;; "050") ntasks=16 @@ -618,18 +615,16 @@ case ${step} in exit 4 esac - walltime="00:10:00" - threads_per_task=1 + walltime="00:30:00" export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; "marineanlchkpt") walltime="00:10:00" ntasks=1 - threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} case ${OCNRES} in "025") memory="128GB" @@ -652,8 +647,7 @@ case ${step} in "marineanlfinal") walltime="00:30:00" ntasks=${max_tasks_per_node} - threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} ;; "ocnanalvrfy") diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a2ea3770ae..92750bedb5 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a2ea3770aeb9d4308bde51bb1d8c9c94cc9534c8 +Subproject commit 92750bedb5ec650fc77fdb305ae6021e23b841af diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index dd48c9f80f..31aece713e 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -206,7 +206,7 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs envconfig_jcb['NMEM_ENS'] = self.task_config.NMEM_ENS envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion' - if self.task_config.NMEM_ENS >= 3: + if self.task_config.NMEM_ENS >= 2: envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion' envconfig_jcb['DATA'] = self.task_config.DATA envconfig_jcb['OPREFIX'] = self.task_config.OPREFIX From b565befef9a6b02b4fc552ddd2a9d368fa5c706f Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 18 Dec 2024 17:46:21 +0000 Subject: [PATCH 02/18] Switch to using updating jedi hashes + soca and jcb-gdas changes for soca2cice --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 92750bedb5..31e477d9b8 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 92750bedb5ec650fc77fdb305ae6021e23b841af +Subproject commit 31e477d9b865a08e01727786ff6d16ba51a1d2ef From e1a4fd4007f7e2eeeae9d2420c08ae2d0fbc5163 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 18 Dec 2024 19:09:02 +0000 Subject: [PATCH 03/18] Bugfix some resources --- parm/config/gfs/config.resources | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index f17eb7cc50..562ea8467a 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -502,6 +502,7 @@ case ${step} in "marineanlinit") walltime="00:10:00" ntasks=1 + threads_per_task=1 tasks_per_node=${max_tasks_per_node} memory="24GB" ;; @@ -509,6 +510,7 @@ case ${step} in "prepoceanobs") walltime="00:10:00" ntasks=1 + threads_per_task=1 tasks_per_node=${max_tasks_per_node} memory="48GB" ;; @@ -518,8 +520,9 @@ case ${step} in ntasks=16 case ${OCNRES} in "025") - ntasks=480;; + ntasks=480 memory="256GB" + ;; "050") ntasks=16;; "100") ntasks=16;; "500") ntasks=16;; @@ -529,6 +532,7 @@ case ${step} in esac walltime="00:30:00" + threads_per_task=1 export is_exclusive=True tasks_per_node=$(( max_tasks_per_node / 2 )) ;; @@ -558,6 +562,7 @@ case ${step} in esac walltime="00:30:00" + threads_per_task=1 export is_exclusive=True tasks_per_node=$(( max_tasks_per_node / 2 )) ;; @@ -588,6 +593,7 @@ case ${step} in walltime="00:10:00" export is_exclusive=True + threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / 2 )) ;; @@ -617,6 +623,7 @@ case ${step} in walltime="00:30:00" export is_exclusive=True + threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / 2 )) ;; @@ -624,6 +631,7 @@ case ${step} in "marineanlchkpt") walltime="00:10:00" ntasks=1 + threads_per_task=1 tasks_per_node=${max_tasks_per_node} case ${OCNRES} in "025") @@ -647,6 +655,7 @@ case ${step} in "marineanlfinal") walltime="00:30:00" ntasks=${max_tasks_per_node} + threads_per_task=1 tasks_per_node=${max_tasks_per_node} ;; From e284329fdc4a67be6185c96089011e20edb39d34 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 18 Dec 2024 19:55:10 +0000 Subject: [PATCH 04/18] change paths to hera + add dmpdir --- ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml | 3 ++- ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml | 3 ++- ci/cases/gfsv17/marine3dvar.yaml | 3 +++ ci/cases/gfsv17/marinehyb.yaml | 3 +++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index c47b53339e..21aece76a5 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -15,5 +15,6 @@ arguments: idate: 2021063018 edate: 2021070306 #icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2 - icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/ yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marine3dvar.yaml diff --git a/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml index f0e0b42c28..57680ffd02 100644 --- a/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml @@ -15,5 +15,6 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021063018 edate: 2021070306 - icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/ yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marinehyb.yaml diff --git a/ci/cases/gfsv17/marine3dvar.yaml b/ci/cases/gfsv17/marine3dvar.yaml index abf86f0aa8..54258b1e4b 100644 --- a/ci/cases/gfsv17/marine3dvar.yaml +++ b/ci/cases/gfsv17/marine3dvar.yaml @@ -17,6 +17,9 @@ base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "YES" +prepoceanobs: + DMPDIR: /scratch1/NCEPDEV/da/common/ + marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml diff --git a/ci/cases/gfsv17/marinehyb.yaml b/ci/cases/gfsv17/marinehyb.yaml index ed62e607a8..f7b94eb971 100644 --- a/ci/cases/gfsv17/marinehyb.yaml +++ b/ci/cases/gfsv17/marinehyb.yaml @@ -16,6 +16,9 @@ base: FHMAX_GFS: 240 ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} +prepoceanobs: + DMPDIR: /scratch1/NCEPDEV/da/common/ + marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca SOCA_NINNER: 20 # revert to ~100 after the memory leak is fixed From 7c37d0c75d36093885cfec5ab509be8e551a3458 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 18 Dec 2024 21:37:30 +0000 Subject: [PATCH 05/18] fix the DMPDIR specification --- ci/cases/gfsv17/marine3dvar.yaml | 3 ++- ci/cases/gfsv17/marinehyb.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/cases/gfsv17/marine3dvar.yaml b/ci/cases/gfsv17/marine3dvar.yaml index 54258b1e4b..24380dffd2 100644 --- a/ci/cases/gfsv17/marine3dvar.yaml +++ b/ci/cases/gfsv17/marine3dvar.yaml @@ -18,7 +18,8 @@ base: DO_TEST_MODE: "YES" prepoceanobs: - DMPDIR: /scratch1/NCEPDEV/da/common/ + use_exp_obs: "YES" + dmpdir_exp: /scratch1/NCEPDEV/da/common/ marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca diff --git a/ci/cases/gfsv17/marinehyb.yaml b/ci/cases/gfsv17/marinehyb.yaml index f7b94eb971..20614ad290 100644 --- a/ci/cases/gfsv17/marinehyb.yaml +++ b/ci/cases/gfsv17/marinehyb.yaml @@ -17,7 +17,8 @@ base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} prepoceanobs: - DMPDIR: /scratch1/NCEPDEV/da/common/ + use_exp_obs: "YES" + dmpdir_exp: /scratch1/NCEPDEV/da/common/ marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca From e95d993283cb0d73e086711821124c111c115c5b Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 9 Jan 2025 09:54:06 -0600 Subject: [PATCH 06/18] ... --- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/ufs_utils.fd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 856a42076a..4848ecbb5e 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 856a42076a65256aaae9b29f4891532cb4a3fbca +Subproject commit 4848ecbb5e713b16127433e11f7d3edc6ac784c4 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 9f44c8798c..27c03e83a6 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 9f44c8798c2087aca06df8f629699632e57df431 +Subproject commit 27c03e83a66fa2ba3b32781dab7cd6a8c0fc497d diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 06eec5b6f6..2323761084 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 06eec5b6f636123835e2dfd9fc5229980c006735 +Subproject commit 23237610845c3a4438b21b25e9b3dc25c4c15b73 From fd54e37fdfd1a391969435542573f08e9bd33fa9 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 15 Jan 2025 15:28:04 -0600 Subject: [PATCH 07/18] low-res bmat works but still wip --- parm/config/gfs/config.marineanl | 1 + parm/config/gfs/yaml/defaults.yaml | 1 + parm/gdas/soca_bmat_jedi_config.yaml.j2 | 30 ++++++++++++++--------- sorc/link_workflow.sh | 1 + ush/python/pygfs/task/marine_bmat.py | 12 +++++++-- ush/python/pygfs/utils/marine_da_utils.py | 4 +-- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index b0fbbedb54..e3dcc4529e 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -11,6 +11,7 @@ export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@ export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ +export SOCA_ANL_GEOM=@SOCA_ANL_GEOM@ export SOCA_NINNER=@SOCA_NINNER@ export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 55f4b03f50..9018142502 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -54,6 +54,7 @@ snowanl: marineanl: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" + SOCA_ANL_GEOM: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/soca/marine-jcb-3dfgat.yaml.j2" diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2 index 4e476d3117..df115e142f 100644 --- a/parm/gdas/soca_bmat_jedi_config.yaml.j2 +++ b/parm/gdas/soca_bmat_jedi_config.yaml.j2 @@ -8,35 +8,41 @@ soca_diagb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x' mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_diagb +soca_chgres: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/soca_convertstate.x' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_chgres soca_parameters_diffusion_vt: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_parameters_diffusion_vt soca_setcorscales: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_setcorscales soca_parameters_diffusion_hz: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_parameters_diffusion_hz soca_ensb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_ens_handler.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_ensb soca_ensweights: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' - jcb_algo: soca_ensweights + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_ensweights diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index b70b9e894f..cc2f9e3ddd 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -364,6 +364,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdas_soca_gridgen.x" "gdas_soca_error_covariance_toolbox.x" "gdas_fv3jedi_error_covariance_toolbox.x" + "soca_convertstate.x" "gdas_soca_setcorscales.x" "gdas_soca_diagb.x" "fv3jedi_plot_field.x" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 8e2b84a673..d4b6526b6e 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -74,7 +74,7 @@ def __init__(self, config): # Create dictionary of Jedi objects expected_keys = ['gridgen', 'soca_diagb', 'soca_parameters_diffusion_vt', 'soca_setcorscales', - 'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights'] + 'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights', 'soca_chgres'] self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) @logit(logger) @@ -105,9 +105,12 @@ def initialize(self: Task) -> None: soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_list).sync() - # prepare the MOM6 input.nml + # prepare the deterministic MOM6 input.nml mdau.prep_input_nml(self.task_config) + # prepare the input.nml for the analysis geometry + mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml") + # stage backgrounds # TODO(G): Check ocean backgrounds dates for consistency bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) @@ -127,6 +130,7 @@ def initialize(self: Task) -> None: # initialize JEDI applications self.jedi_dict['gridgen'].initialize(self.task_config) self.jedi_dict['soca_diagb'].initialize(self.task_config) + self.jedi_dict['soca_chgres'].initialize(self.task_config) self.jedi_dict['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi_dict['soca_setcorscales'].initialize(self.task_config) self.jedi_dict['soca_parameters_diffusion_hz'].initialize(self.task_config) @@ -163,11 +167,15 @@ def execute(self) -> None: None """ + # soca grid generation self.jedi_dict['gridgen'].execute() # variance partitioning self.jedi_dict['soca_diagb'].execute() + # Interpolate f009 bkg to analysis geometry + self.jedi_dict['soca_chgres'].execute() + # horizontal diffusion self.jedi_dict['soca_setcorscales'].execute() self.jedi_dict['soca_parameters_diffusion_hz'].execute() diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 50d9d84e86..9e3ee5e2b3 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -45,7 +45,7 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None: @logit(logger) -def prep_input_nml(task_config: AttrDict) -> None: +def prep_input_nml(task_config: AttrDict, output_nml = "mom_input.nml") -> None: """Prepare the mom_input.nml file """ # stage input.nml.j2 @@ -58,7 +58,7 @@ def prep_input_nml(task_config: AttrDict) -> None: input_nml_config = {'domain_stack_size': task_config.DOMAIN_STACK_SIZE, 'date_init': date_init} jinja_input_nml = jinja.Jinja(mom_input_nml_tmpl, input_nml_config) - jinja_input_nml.save('mom_input.nml') + jinja_input_nml.save(output_nml) @logit(logger) From 7c2c058c1818f92f8628daffb0db0e84cbd3cb3f Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 16 Jan 2025 08:57:54 -0600 Subject: [PATCH 08/18] B geom in analysis task --- ush/python/pygfs/task/marine_analysis.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 31aece713e..3ccaec9c25 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -179,9 +179,12 @@ def _prep_scratch_dir(self: Task) -> None: soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_list).sync() - # prepare the MOM6 input.nml + # prepare the deterministic MOM6 input.nml mdau.prep_input_nml(self.task_config) + # prepare the input.nml for the analysis geometry + mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml") + # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}") soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) @@ -364,7 +367,7 @@ def list_all_files(dir_in, dir_out, wc='*', fh_list=[]): post_file_list = [] # Make a copy the IAU increment - post_file_list.append([os.path.join(anl_dir, 'inc.nc'), + post_file_list.append([os.path.join(anl_dir, 'ocn.inc.nc'), os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocninc.nc')]) domains = ['ocn', 'ice'] From e62e9d090aae35943a39a68b38cee7ec5429a953 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 16 Jan 2025 12:18:34 -0600 Subject: [PATCH 09/18] use gdas.x for chg res --- ci/cases/gfsv17/marine3dvar.yaml | 1 + ci/cases/gfsv17/marinehyb.yaml | 3 ++- parm/gdas/soca_bmat_jedi_config.yaml.j2 | 1 + sorc/link_workflow.sh | 1 - 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/cases/gfsv17/marine3dvar.yaml b/ci/cases/gfsv17/marine3dvar.yaml index 24380dffd2..da4d4fb599 100644 --- a/ci/cases/gfsv17/marine3dvar.yaml +++ b/ci/cases/gfsv17/marine3dvar.yaml @@ -23,5 +23,6 @@ prepoceanobs: marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca + SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 100 diff --git a/ci/cases/gfsv17/marinehyb.yaml b/ci/cases/gfsv17/marinehyb.yaml index 20614ad290..72515088e8 100644 --- a/ci/cases/gfsv17/marinehyb.yaml +++ b/ci/cases/gfsv17/marinehyb.yaml @@ -22,4 +22,5 @@ prepoceanobs: marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca - SOCA_NINNER: 20 # revert to ~100 after the memory leak is fixed + SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca + SOCA_NINNER: 100 diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2 index df115e142f..59e7e1d056 100644 --- a/parm/gdas/soca_bmat_jedi_config.yaml.j2 +++ b/parm/gdas/soca_bmat_jedi_config.yaml.j2 @@ -14,6 +14,7 @@ soca_chgres: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/soca_convertstate.x' mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jedi_args: ['soca', 'convertstate'] jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_chgres soca_parameters_diffusion_vt: diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f4b6e41f17..1988fe60f6 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -361,7 +361,6 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdas_soca_gridgen.x" "gdas_soca_error_covariance_toolbox.x" "gdas_fv3jedi_error_covariance_toolbox.x" - "soca_convertstate.x" "gdas_soca_setcorscales.x" "gdas_soca_diagb.x" "fv3jedi_plot_field.x" From 069911603c06eadea169bfbe66489f152a3c6659 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 16 Jan 2025 12:50:54 -0600 Subject: [PATCH 10/18] norms --- ush/python/pygfs/utils/marine_da_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 9e3ee5e2b3..67d4f9a991 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -45,7 +45,7 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None: @logit(logger) -def prep_input_nml(task_config: AttrDict, output_nml = "mom_input.nml") -> None: +def prep_input_nml(task_config: AttrDict, output_nml="mom_input.nml") -> None: """Prepare the mom_input.nml file """ # stage input.nml.j2 From d736972ab1fac4badd1b7dbbb5bb51869e831113 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 16 Jan 2025 14:36:10 -0600 Subject: [PATCH 11/18] testing hack, revert when new fix files are staged --- sorc/link_workflow.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 1988fe60f6..48a706effd 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -213,6 +213,10 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then fix_ver="gdas_${gdas_sub}_ver" ${LINK_OR_COPY} "${FIX_DIR}/gdas/${gdas_sub}/${!fix_ver}" "${gdas_sub}" done + # HACK, only for testing,don't merge. + rm soca + ln -s /work2/noaa/da/gvernier/prs/soca . + # END HACK fi #------------------------------ From 0c5ed8713c624c08c0135aea6183e8b72b40dae9 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 16 Jan 2025 17:49:20 -0600 Subject: [PATCH 12/18] fixed wrong exec --- parm/gdas/soca_bmat_jedi_config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2 index 59e7e1d056..9517626b98 100644 --- a/parm/gdas/soca_bmat_jedi_config.yaml.j2 +++ b/parm/gdas/soca_bmat_jedi_config.yaml.j2 @@ -12,7 +12,7 @@ soca_diagb: jcb_algo: soca_diagb soca_chgres: rundir: '{{ DATA }}' - exe_src: '{{ EXECgfs }}/soca_convertstate.x' + exe_src: '{{ EXECgfs }}/gdas.x' mpi_cmd: '{{ APRUN_MARINEBMAT }}' jedi_args: ['soca', 'convertstate'] jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' From 29e0c782286c10ec49a7eace5b830c4e66d17e4d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 21 Jan 2025 10:10:56 -0600 Subject: [PATCH 13/18] bugfix of low res geom --- ush/python/pygfs/task/marine_analysis.py | 3 ++- ush/python/pygfs/task/marine_bmat.py | 3 ++- ush/python/pygfs/utils/marine_da_utils.py | 9 +++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 3ccaec9c25..bd61d8bc5a 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -183,7 +183,8 @@ def _prep_scratch_dir(self: Task) -> None: mdau.prep_input_nml(self.task_config) # prepare the input.nml for the analysis geometry - mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml") + mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml", + simple_geom=True, mom_input="./anl_geom/MOM_input") # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}") diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index d4b6526b6e..e64c1527c3 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -109,7 +109,8 @@ def initialize(self: Task) -> None: mdau.prep_input_nml(self.task_config) # prepare the input.nml for the analysis geometry - mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml") + mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml", + simple_geom=True, mom_input="./anl_geom/MOM_input") # stage backgrounds # TODO(G): Check ocean backgrounds dates for consistency diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 67d4f9a991..e50534c0ee 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -45,7 +45,10 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None: @logit(logger) -def prep_input_nml(task_config: AttrDict, output_nml="mom_input.nml") -> None: +def prep_input_nml(task_config: AttrDict, + output_nml: str="mom_input.nml", + simple_geom: bool=False, + mom_input: str="MOM_input") -> None: """Prepare the mom_input.nml file """ # stage input.nml.j2 @@ -56,7 +59,9 @@ def prep_input_nml(task_config: AttrDict, output_nml="mom_input.nml") -> None: # swap date and stacksize date_init = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')] input_nml_config = {'domain_stack_size': task_config.DOMAIN_STACK_SIZE, - 'date_init': date_init} + 'date_init': date_init, + 'simple_geom': simple_geom, + 'mom_input': mom_input} jinja_input_nml = jinja.Jinja(mom_input_nml_tmpl, input_nml_config) jinja_input_nml.save(output_nml) From 3303119fb3866198cdde5688daa37d38ac77edb6 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 21 Jan 2025 10:18:07 -0600 Subject: [PATCH 14/18] fixed pynorms --- ush/python/pygfs/utils/marine_da_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index e50534c0ee..748b5ff9c9 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -46,9 +46,9 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None: @logit(logger) def prep_input_nml(task_config: AttrDict, - output_nml: str="mom_input.nml", - simple_geom: bool=False, - mom_input: str="MOM_input") -> None: + output_nml: str = "mom_input.nml", + simple_geom: bool = False, + mom_input: str = "MOM_input") -> None: """Prepare the mom_input.nml file """ # stage input.nml.j2 From 534ab0851ce5a2eefd274f0036aa984ed2042674 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 22 Jan 2025 09:21:34 -0600 Subject: [PATCH 15/18] removed hack --- sorc/link_workflow.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 48a706effd..1988fe60f6 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -213,10 +213,6 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then fix_ver="gdas_${gdas_sub}_ver" ${LINK_OR_COPY} "${FIX_DIR}/gdas/${gdas_sub}/${!fix_ver}" "${gdas_sub}" done - # HACK, only for testing,don't merge. - rm soca - ln -s /work2/noaa/da/gvernier/prs/soca . - # END HACK fi #------------------------------ From c90ba0c43da2d1ec565c396e3885780065135957 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 27 Jan 2025 12:50:50 -0600 Subject: [PATCH 16/18] ... --- sorc/gsi_monitor.fd | 2 +- sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 879f5900d9..278ee629e8 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 879f5900d9236e144a18cba8e804618fa6c449e3 +Subproject commit 278ee629e87558822e8d13b3fb3b0e16006aa856 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 76471dc6b7..63ace62a36 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 76471dc6b7bfc3342416d1a3402f360724f7c0fa +Subproject commit 63ace62a36a263f03b914a92fc5536509e862dbc diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 5b2bfa39a5..2323761084 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 5b2bfa39a5b1fbe6bb1d247102679885685c9336 +Subproject commit 23237610845c3a4438b21b25e9b3dc25c4c15b73 From 9a693bf10a81855d3cd4e941cdd4b0195672032c Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 27 Jan 2025 13:09:52 -0600 Subject: [PATCH 17/18] ... --- sorc/gsi_monitor.fd | 2 +- sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 278ee629e8..879f5900d9 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 278ee629e87558822e8d13b3fb3b0e16006aa856 +Subproject commit 879f5900d9236e144a18cba8e804618fa6c449e3 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 63ace62a36..76471dc6b7 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 63ace62a36a263f03b914a92fc5536509e862dbc +Subproject commit 76471dc6b7bfc3342416d1a3402f360724f7c0fa diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 2323761084..5b2bfa39a5 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 23237610845c3a4438b21b25e9b3dc25c4c15b73 +Subproject commit 5b2bfa39a5b1fbe6bb1d247102679885685c9336 From d93a95c47b14813ac1b46cdd7b1086d118fbc65e Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 28 Jan 2025 14:37:06 -0600 Subject: [PATCH 18/18] updated soca fix version --- versions/fix.ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/fix.ver b/versions/fix.ver index db54701e77..d6e7027685 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -9,7 +9,7 @@ export cpl_ver=20230526 export datm_ver=20220805 export gdas_crtm_ver=20220805 export gdas_fv3jedi_ver=20241115 -export gdas_soca_ver=20240802 +export gdas_soca_ver=20240919 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 export gdas_aero_ver=20240806